Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.26728258 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.018173097 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.018876152 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.859026656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.923931848Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.925007313Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.925049104Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.925063975Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.925079545Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188} {"msg":"Crucible 1e977c31-ea08-4c9a-8e4e-45e1fe8700f4 has session id: 89ddf313-befa-42c0-80da-5b3f1763d6cb","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.925844373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Upstairs opts: Upstairs UUID: 1e977c31-ea08-4c9a-8e4e-45e1fe8700f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.925910665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Crucible stats registered with UUID: 1e977c31-ea08-4c9a-8e4e-45e1fe8700f4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.925930295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961094716Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961144017Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188} {"msg":"1e977c31-ea08-4c9a-8e4e-45e1fe8700f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961216509Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.96124901Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961311661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961328662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961342932Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961356562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961370652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961384393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961399243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961424204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:53275)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961558887Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56218)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961602348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:49083)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.961884915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.963382021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.963412391Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.963427882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.963442272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.963460353Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.963480923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.964529408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.964568999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.9645969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965245135Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965273156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965287906Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965387808Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965425969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.96544933Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.96547027Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965491911Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965546762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965574463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965594264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965651135Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965670315Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965684736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965698666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965712376Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965727207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965740597Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965754327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965767818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965837859Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"downstairs","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.96585738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.96587016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.96588448Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965897751Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"client":"2","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965914141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965926792Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"1e977c31-ea08-4c9a-8e4e-45e1fe8700f4 is now active with session: 89ddf313-befa-42c0-80da-5b3f1763d6cb","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965939782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965956212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:40.965979403Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.054059775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188} {"msg":"client_stop_rx closed unexpectedly: RecvError(())","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.054115426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.054148297Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.054183507Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.054218188Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.054250949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.05429503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.054328591Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.054357771Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"89ddf313-befa-42c0-80da-5b3f1763d6cb"} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.08483617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.134101708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.134751413Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.134788654Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.134803354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.134817735Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190} {"msg":"Crucible aada99c3-ba45-4666-8690-eb92a3bc09f1 has session id: 447feae3-7b33-4ed8-8dfe-829d56f0aa37","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.135257665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Upstairs opts: Upstairs UUID: aada99c3-ba45-4666-8690-eb92a3bc09f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.135303746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Crucible stats registered with UUID: aada99c3-ba45-4666-8690-eb92a3bc09f1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.135319447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165717684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165756725Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190} {"msg":"aada99c3-ba45-4666-8690-eb92a3bc09f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165819126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165847647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165875617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165899438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165928179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.165952429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.16597377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.16599672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166024841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166053692Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_connection connected from Ok(127.0.0.1:62812)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166079482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_connection connected from Ok(127.0.0.1:54344)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166109243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_connection connected from Ok(127.0.0.1:54703)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166140144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166782369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.16680937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166945083Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.166980684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.167008995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.167033675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.167435965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.167479586Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.167755182Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168263165Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168305126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168335536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168395858Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168423258Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168438289Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168451319Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168465079Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.16850626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168527951Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168543141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168556992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168570672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168584692Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168598173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168611693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168625723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168638974Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168652174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168665174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:07:45.168678224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"downstairs","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168704855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168730316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"0","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168759156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168785917Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"client":"2","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:07:45.168811938Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168839338Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"aada99c3-ba45-4666-8690-eb92a3bc09f1 is now active with session: 447feae3-7b33-4ed8-8dfe-829d56f0aa37","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168864149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168878349Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.168890809Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 173 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Read block 20 len 12288 03/30 Read block 330 len 12288 04/30 Read block 536 len 24576 05/30 Write block 301 len 8192 data: 2 2 06/30 Read block 180 len 36864 07/30 Flush 08/30 Write block 24 len 8192 data: 2 2 09/30 Read block 44 len 20480 10/30 Write block 344 len 4096 data: 2 11/30 Write block 392 len 24576 data: 2 2 2 2 2 2 12/30 Read block 46 len 36864 13/30 Read block 403 len 40960 14/30 Write block 141 len 32768 data: 2 2 2 2 2 2 2 2 15/30 Flush 16/30 Flush 17/30 Write block 61 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Write block 52 len 4096 data: 2 19/30 Write block 389 len 20480 data: 2 2 2 3 3 20/30 Read block 517 len 8192 21/30 Read block 316 len 4096 22/30 Read block 88 len 16384 23/30 Read block 521 len 16384 24/30 Read block 171 len 4096 25/30 Read block 290 len 28672 26/30 Flush 27/30 Write block 356 len 4096 data: 2 28/30 Write block 267 len 36864 data: 2 2 2 2 2 2 2 2 2 29/30 Write block 422 len 32768 data: 2 2 2 2 2 2 2 2 30/30 Write block 272 len 16384 data: 3 3 3 3 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.187675709Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190} Wrote out file "/tmp/test_repair_verify.data" {"msg":"client task was cancelled without us; hopefully the program is exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.188384535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"client task is sending Done(ReceiveTaskCancelled)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.188428347Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:45.188455317Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:45.188483808Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"447feae3-7b33-4ed8-8dfe-829d56f0aa37"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 5 5 5 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 T T T 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 3 3 3 T T T 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 T T T 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 5 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 5 5 5 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 3 3 3 6 6 6 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 3 3 3 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 6 6 6 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.443878659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.486829766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.487456001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.487492212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.487506992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.487521392Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198} {"msg":"Crucible fd654d45-d78a-402f-80ef-5b9f48d41f43 has session id: a67c6405-f97a-4b81-b3cd-2476b1435a77","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.487951132Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Upstairs opts: Upstairs UUID: fd654d45-d78a-402f-80ef-5b9f48d41f43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.487996773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Crucible stats registered with UUID: fd654d45-d78a-402f-80ef-5b9f48d41f43","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.488022224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517711104Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517737784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198} {"msg":"fd654d45-d78a-402f-80ef-5b9f48d41f43 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517810466Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517833537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517859627Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517874288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517887918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517901218Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517923109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517943089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.51796131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.517987281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_connection connected from Ok(127.0.0.1:38824)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.518087633Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_connection connected from Ok(127.0.0.1:48585)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.518281377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_connection connected from Ok(127.0.0.1:35981)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.518326418Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.51881037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.518841201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.518857531Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.518873932Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.518996634Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.519042016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.519581668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.519612229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.519720272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520134232Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520174073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520192373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520205663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520219604Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520233084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520245455Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520258815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[0]R flush_numbers[0..12]: [1, 3, 5, 5, 1, 1, 1, 3, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520272675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[0]R generation[0..12]: [1, 2, 2, 2, 1, 1, 1, 2, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520286696Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520304176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[1]R flush_numbers[0..12]: [1, 3, 5, 5, 1, 1, 1, 3, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520318306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[1]R generation[0..12]: [1, 2, 2, 2, 1, 1, 1, 2, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520332266Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520346537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[2]R flush_numbers[0..12]: [1, 3, 5, 5, 1, 1, 1, 3, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520360067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[2]R generation[0..12]: [1, 2, 2, 2, 1, 1, 1, 2, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520373827Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520387978Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520401238Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520414648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520427709Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520448579Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"":"downstairs","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.52046209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.52047446Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"0","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.52048789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"1","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520501321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"client":"2","":"downstairs","session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520516651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520529541Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"fd654d45-d78a-402f-80ef-5b9f48d41f43 is now active with session: a67c6405-f97a-4b81-b3cd-2476b1435a77","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520542431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520554612Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198,"session_id":"a67c6405-f97a-4b81-b3cd-2476b1435a77"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:49.520567022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 20480 data: 2 2 2 2 2 02/30 Write block 53 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 307 len 4096 04/30 Write block 37 len 40960 data: 2 2 2 2 2 2 2 2 2 2 05/30 Write block 82 len 4096 data: 2 06/30 Read block 162 len 12288 07/30 Write block 390 len 24576 data: 3 3 4 4 3 3 08/30 Flush 09/30 Write block 66 len 24576 data: 3 3 3 3 3 2 10/30 Write block 561 len 20480 data: 2 2 2 2 2 11/30 Read block 72 len 20480 12/30 Write block 579 len 12288 data: 2 2 2 13/30 Read block 380 len 32768 14/30 Write block 313 len 20480 data: 2 2 2 2 2 15/30 Write block 30 len 8192 data: 2 2 16/30 Write block 225 len 4096 data: 2 17/30 Write block 316 len 24576 data: 3 3 2 2 2 2 18/30 Write block 484 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Read block 484 len 8192 20/30 Write block 319 len 40960 data: 3 3 3 2 2 2 2 2 2 2 21/30 Read block 56 len 40960 22/30 Flush 23/30 Write block 517 len 36864 data: 2 2 2 2 2 2 2 2 2 24/30 Read block 419 len 24576 25/30 Read block 205 len 40960 26/30 Flush 27/30 Write block 394 len 20480 data: 4 4 3 3 2 28/30 Read block 74 len 12288 29/30 Read block 108 len 24576 30/30 Write block 131 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:49.539418663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 8 8 8 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 4 4 4 8 8 8 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 T T T 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 4 4 4 8 8 8 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 3 3 3 6 6 6 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 3 3 3 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 4 4 4 7 7 7 T T T 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 6 6 6 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 4 4 4 9 9 9 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 4 4 4 8 8 8 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 4, Max flush: 9 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 8 8 8 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 4 4 4 8 8 8 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 5 5 5 10 10 10 F F F 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 4 4 4 8 8 8 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 3 3 3 6 6 6 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 3 3 3 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 6 6 6 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 4 4 4 9 9 9 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 4 4 4 8 8 8 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.777797416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.821060021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.821704086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.821743177Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.821759357Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.821774208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206} {"msg":"Crucible c79d2d52-6735-4939-8386-20f18c7bd426 has session id: 0c0ac25e-3eaa-498e-a8c5-589c603e6bc7","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.822261389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Upstairs opts: Upstairs UUID: c79d2d52-6735-4939-8386-20f18c7bd426, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.82229774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Crucible stats registered with UUID: c79d2d52-6735-4939-8386-20f18c7bd426","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.822322351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852046021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852079242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206} {"msg":"c79d2d52-6735-4939-8386-20f18c7bd426 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852128193Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852149044Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852177525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852195765Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852213595Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852242446Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852308728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852345658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852376479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852516633Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46841)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852561744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51085)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852596144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62489)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.852627065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853142828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853180718Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853210759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.85324042Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853269581Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853286881Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853906736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853942397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.853976937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854463179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.8544932Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.8545089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854521701Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854535501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854548981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854561691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854575372Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[0]R flush_numbers[0..12]: [7, 8, 7, 8, 7, 1, 10, 3, 2, 2, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854589262Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 4, 4, 1, 5, 2, 2, 2, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854603052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854617313Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[1]R flush_numbers[0..12]: [7, 8, 7, 8, 7, 1, 10, 3, 2, 2, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854631003Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 4, 4, 1, 5, 2, 2, 2, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854644644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854658754Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[2]R flush_numbers[0..12]: [7, 8, 7, 8, 7, 1, 10, 3, 2, 2, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854676204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 4, 4, 1, 5, 2, 2, 2, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854690765Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:07:53.854704925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854725555Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854738936Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854751926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854765016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"":"downstairs","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854778327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854790657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"0","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854804237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"1","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854817568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"client":"2","":"downstairs","session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854844568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854859009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"c79d2d52-6735-4939-8386-20f18c7bd426 is now active with session: 0c0ac25e-3eaa-498e-a8c5-589c603e6bc7","v":0,"name":"crucible","level":30,"time"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"2024-12-09T15:07:53.854872049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.854888259Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206,"session_id":"0c0ac25e-3eaa-498e-a8c5-589c603e6bc7"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:53.85490073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 12288 data: 2 2 2 02/30 Flush 03/30 Write block 229 len 4096 data: 2 04/30 Write block 222 len 28672 data: 2 2 2 3 2 2 2 05/30 Read block 208 len 20480 06/30 Read block 171 len 12288 07/30 Flush 08/30 Flush 09/30 Write block 140 len 20480 data: 2 3 3 3 3 10/30 Read block 418 len 8192 11/30 Read block 141 len 4096 12/30 Read block 509 len 40960 13/30 Write block 173 len 4096 data: 3 14/30 Read block 2 len 16384 15/30 Write block 61 len 28672 data: 3 3 3 3 3 4 4 16/30 Write block 418 len 24576 data: 2 2 2 2 3 3 17/30 Write block 224 len 16384 data: 3 4 3 3 18/30 Read block 223 len 36864 19/30 Write block 404 len 4096 data: 2 20/30 Write block 164 len 12288 data: 2 2 2 21/30 Flush 22/30 Flush 23/30 Write block 573 len 36864 data: 2 2 2 2 2 2 3 3 3 24/30 Write block 260 len 4096 data: 2 25/30 Read block 174 len 16384 26/30 Read block 394 len 36864 27/30 Read block 235 len 8192 28/30 Write block 102 len 8192 data: 2 2 29/30 Read block 53 len 20480 30/30 Write block 553 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:53.871831074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 8 8 8 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 6 6 6 14 14 14 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 5 5 5 10 10 10 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 6 6 6 14 14 14 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 3 3 3 6 6 6 T T T 14 280-299 1 1 1 1 1 1 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 3 3 3 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 6 6 6 14 14 14 F F F 21 420-439 6 6 6 14 14 14 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 4 4 4 9 9 9 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 4 4 4 8 8 8 T T T 29 580-599 4 4 4 8 8 8 T T T Max gen: 6, Max flush: 14 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 8 8 8 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 6 6 6 14 14 14 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 7 7 7 15 15 15 F F F 6 120-139 5 5 5 10 10 10 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 6 6 6 14 14 14 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 7 7 7 15 15 15 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 3 3 3 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 6 6 6 14 14 14 F F F 21 420-439 6 6 6 14 14 14 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 4 4 4 9 9 9 F F F 27 540-559 7 7 7 15 15 15 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 7 7 7 15 15 15 F F F Max gen: 7, Max flush: 15 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.132184344Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.176486873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.177272812Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.177310922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.177326003Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.177340793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214} {"msg":"Crucible d31cf9a5-2fc8-4b3d-9b83-2bef86076519 has session id: d6dfb3e8-ee55-4a75-accd-af82c53b3de6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.177788624Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Upstairs opts: Upstairs UUID: d31cf9a5-2fc8-4b3d-9b83-2bef86076519, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.177838015Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Crucible stats registered with UUID: d31cf9a5-2fc8-4b3d-9b83-2bef86076519","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.177859546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208443997Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208499118Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214} {"msg":"d31cf9a5-2fc8-4b3d-9b83-2bef86076519 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208530679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208560929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208611481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208648812Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208683452Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208717483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208752944Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208789205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208813046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208831116Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58420)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208851116Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59592)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208888297Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_connection connected from Ok(127.0.0.1:65038)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.208920018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.209735438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.209776558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.209803009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.20982585Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.209922842Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.209958003Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.210506026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.210537797Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.210555237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211108701Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211137831Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211153071Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211216823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211251034Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211268124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211281485Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211295385Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[0]R flush_numbers[0..12]: [7, 8, 7, 14, 7, 15, 10, 14, 14, 2, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211309475Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 6, 4, 7, 5, 6, 6, 2, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211323975Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211338576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[1]R flush_numbers[0..12]: [7, 8, 7, 14, 7, 15, 10, 14, 14, 2, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211352726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 6, 4, 7, 5, 6, 6, 2, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211366427Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211380807Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[2]R flush_numbers[0..12]: [7, 8, 7, 14, 7, 15, 10, 14, 14, 2, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211394657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 6, 4, 7, 5, 6, 6, 2, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211418698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211448538Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211470469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211490559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.21152986Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211558561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"":"downstairs","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211574561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211586982Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"0","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211608562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"1","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211635283Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"client":"2","":"downstairs","session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211662674Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211677304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"d31cf9a5-2fc8-4b3d-9b83-2bef86076519 is now active with session: d6dfb3e8-ee55-4a75-accd-af82c53b3de6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211696815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211720455Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214,"session_id":"d6dfb3e8-ee55-4a75-accd-af82c53b3de6"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:07:58.211746316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 20 len 12288 data: 2 2 2 02/30 Read block 590 len 20480 03/30 Flush 04/30 Write block 72 len 32768 data: 2 2 2 2 2 2 2 2 05/30 Read block 125 len 32768 06/30 Read block 393 len 32768 07/30 Write block 537 len 28672 data: 2 2 2 2 2 2 2 08/30 Read block 493 len 4096 09/30 Write block 417 len 28672 data: 2 3 3 3 3 4 4 10/30 Write block 506 len 20480 data: 2 2 2 2 2 11/30 Read block 317 len 16384 12/30 Flush 13/30 Write block 336 len 32768 data: 2 2 2 2 2 2 2 2 14/30 Read block 595 len 20480 15/30 Read block 577 len 40960 16/30 Read block 73 len 24576 17/30 Read block 254 len 28672 18/30 Read block 365 len 24576 19/30 Write block 197 len 4096 data: 2 20/30 Flush 21/30 Write block 127 len 40960 data: 2 2 2 2 3 3 3 3 3 3 22/30 Read block 515 len 40960 23/30 Read block 127 len 36864 24/30 Write block 290 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Write block 453 len 4096 data: 2 26/30 Read block 213 len 16384 27/30 Read block 65 len 24576 28/30 Write block 113 len 28672 data: 2 2 2 2 2 2 2 29/30 Read block 474 len 8192 30/30 Write block 314 len 24576 data: 3 3 4 4 3 4 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:07:58.233089716Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 8 8 8 17 17 17 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 7 7 7 15 15 15 T T T 6 120-139 5 5 5 10 10 10 T T T 7 140-159 6 6 6 14 14 14 F F F 8 160-179 6 6 6 14 14 14 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 7 7 7 15 15 15 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 4 4 4 8 8 8 T T T 16 320-339 8 8 8 18 18 18 F F F 17 340-359 8 8 8 18 18 18 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 8 8 8 17 17 17 F F F 21 420-439 8 8 8 17 17 17 F F F 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 7 7 7 15 15 15 F F F Max gen: 8, Max flush: 18 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 8 8 8 17 17 17 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 9 9 9 19 19 19 F F F 6 120-139 9 9 9 19 19 19 F F F 7 140-159 6 6 6 14 14 14 F F F 8 160-179 6 6 6 14 14 14 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 7 7 7 15 15 15 F F F 14 280-299 9 9 9 19 19 19 F F F 15 300-319 9 9 9 19 19 19 F F F 16 320-339 8 8 8 18 18 18 F F F 17 340-359 8 8 8 18 18 18 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 8 8 8 17 17 17 F F F 21 420-439 8 8 8 17 17 17 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 7 7 7 15 15 15 F F F Max gen: 9, Max flush: 19 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.496177761Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.539128598Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.539750313Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.539787263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.539801914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.539816714Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222} {"msg":"Crucible af1b5a49-38f5-43d9-bc81-c533c01fe558 has session id: ac1ff1ec-ee5c-4935-8ebf-294710e80b30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.540204893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Upstairs opts: Upstairs UUID: af1b5a49-38f5-43d9-bc81-c533c01fe558, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.540253225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Crucible stats registered with UUID: af1b5a49-38f5-43d9-bc81-c533c01fe558","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.540279035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570432036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570479287Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222} {"msg":"af1b5a49-38f5-43d9-bc81-c533c01fe558 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570549609Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570569159Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.57059345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.57060842Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570624731Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570650591Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570678082Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570705853Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570736503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570769074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_connection connected from Ok(127.0.0.1:64841)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570804265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_connection connected from Ok(127.0.0.1:55436)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570839196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_connection connected from Ok(127.0.0.1:38693)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.570871046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.571546242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.571588734Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.571667065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.571712086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.571743087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.571775958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572203368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572365732Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572393663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572737371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572779282Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572807013Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572930766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572951996Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572967216Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572980277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.572993837Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[0]R flush_numbers[0..12]: [7, 16, 7, 17, 7, 19, 19, 14, 14, 18, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573007828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[0]R generation[0..12]: [4, 8, 4, 8, 4, 9, 9, 6, 6, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573026158Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573041648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[1]R flush_numbers[0..12]: [7, 16, 7, 17, 7, 19, 19, 14, 14, 18, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573055979Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[1]R generation[0..12]: [4, 8, 4, 8, 4, 9, 9, 6, 6, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573070119Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573084389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[2]R flush_numbers[0..12]: [7, 16, 7, 17, 7, 19, 19, 14, 14, 18, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.57309832Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[2]R generation[0..12]: [4, 8, 4, 8, 4, 9, 9, 6, 6, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.57311214Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.57312637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573139571Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573153141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:08:02.573166321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573186622Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"":"downstairs","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573200212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573212472Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"0","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573227923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"1","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573242063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"client":"2","":"downstairs","session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573255843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573268404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"af1b5a49-38f5-43d9-bc81-c533c01fe558 is now active with session: ac1ff1ec-ee5c-4935-8ebf-294710e80b30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573281344Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573293554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222,"session_id":"ac1ff1ec-ee5c-4935-8ebf-294710e80b30"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:02.573305945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 122 len 24576 data: 2 2 2 2 2 3 02/30 Write block 64 len 16384 data: 4 4 5 5 03/30 Read block 46 len 4096 04/30 Read block 442 len 20480 05/30 Read block 591 len 4096 06/30 Read block 447 len 16384 07/30 Write block 338 len 28672 data: 3 3 3 3 3 3 3 08/30 Read block 216 len 40960 09/30 Read block 147 len 16384 10/30 Read block 315 len 36864 11/30 Flush 12/30 Write block 60 len 16384 data: 2 4 4 4 13/30 Read block 97 len 28672 14/30 Write block 398 len 28672 data: 3 2 2 2 2 2 3 15/30 Read block 534 len 24576 16/30 Read block 399 len 4096 17/30 Write block 95 len 32768 data: 2 2 2 2 2 2 2 3 18/30 Read block 176 len 4096 19/30 Write block 455 len 4096 data: 2 20/30 Read block 446 len 36864 21/30 Write block 453 len 12288 data: 3 2 3 22/30 Write block 462 len 40960 data: 2 2 2 2 2 2 2 2 2 2 23/30 Read block 98 len 20480 24/30 Write block 152 len 40960 data: 2 2 2 2 2 2 2 3 3 3 25/30 Read block 123 len 12288 26/30 Write block 255 len 20480 data: 2 2 2 2 2 27/30 Read block 353 len 20480 28/30 Read block 37 len 4096 29/30 Write block 14 len 16384 data: 2 2 2 2 30/30 Read block 411 len 32768 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 19 19 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:02.593210111Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 T T T 1 020-039 8 8 8 16 16 16 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 10 10 10 20 20 20 T T T 4 080-099 4 4 4 7 7 7 T T T 5 100-119 9 9 9 19 19 19 T T T 6 120-139 10 10 10 20 20 20 F F F 7 140-159 6 6 6 14 14 14 T T T 8 160-179 6 6 6 14 14 14 T T T 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 1 1 1 1 1 1 T T T 13 260-279 7 7 7 15 15 15 F F F 14 280-299 9 9 9 19 19 19 F F F 15 300-319 9 9 9 19 19 19 F F F 16 320-339 10 10 10 20 20 20 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 5 5 5 10 10 10 T T T 20 400-419 8 8 8 17 17 17 T T T 21 420-439 8 8 8 17 17 17 F F F 22 440-459 9 9 9 19 19 19 T T T 23 460-479 1 1 1 1 1 1 T T T 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 7 7 7 15 15 15 F F F Max gen: 10, Max flush: 20 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 21 21 21 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 11 11 11 21 21 21 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 14 14 14 F F F 12 240-259 11 11 11 21 21 21 F F F 13 260-279 7 7 7 15 15 15 F F F 14 280-299 9 9 9 19 19 19 F F F 15 300-319 9 9 9 19 19 19 F F F 16 320-339 10 10 10 20 20 20 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 11 11 11 21 21 21 F F F 21 420-439 8 8 8 17 17 17 F F F 22 440-459 11 11 11 21 21 21 F F F 23 460-479 11 11 11 21 21 21 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 7 7 7 15 15 15 F F F Max gen: 11, Max flush: 21 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.881804034Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:06.923302966Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.923663105Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.954695447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.955271501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.955306682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.955320862Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.955335422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230} {"msg":"Crucible 13f0f3c8-044f-4c7c-8387-0090683126b7 has session id: 67447857-40cc-4dc5-8c33-1b405805f4fe","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.955709001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Upstairs opts: Upstairs UUID: 13f0f3c8-044f-4c7c-8387-0090683126b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.955761793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Crucible stats registered with UUID: 13f0f3c8-044f-4c7c-8387-0090683126b7","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.955779883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.987950332Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.987982373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230} {"msg":"13f0f3c8-044f-4c7c-8387-0090683126b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988037814Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988060145Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988088885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988106456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988123266Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988151447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988185257Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988216418Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988237009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.98826821Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:35124)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.98830217Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:35668)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988337211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:52889)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.988512785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989206152Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989234563Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989253503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989267803Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989282574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989296554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989898378Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.989927659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.98994895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990487763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990517883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990533904Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990617096Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990637536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990651876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990664307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990677877Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[0]R flush_numbers[0..12]: [21, 16, 7, 21, 21, 21, 20, 21, 21, 18, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990691768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[0]R generation[0..12]: [11, 8, 4, 11, 11, 11, 10, 11, 11, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990709538Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990724758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[1]R flush_numbers[0..12]: [21, 16, 7, 21, 21, 21, 20, 21, 21, 18, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990738758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[1]R generation[0..12]: [11, 8, 4, 11, 11, 11, 10, 11, 11, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990752639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990766619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[2]R flush_numbers[0..12]: [21, 16, 7, 21, 21, 21, 20, 21, 21, 18, 1, 14]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990781019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[2]R generation[0..12]: [11, 8, 4, 11, 11, 11, 10, 11, 11, 8, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.99079483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.99080917Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.9908224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990836041Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Next flush: 22","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990906482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990939833Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"":"downstairs","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990957694Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990971214Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"0","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.990988764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"1","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.991003325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"client":"2","":"downstairs","session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.991017045Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.991029785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"13f0f3c8-044f-4c7c-8387-0090683126b7 is now active with session: 67447857-40cc-4dc5-8c33-1b405805f4fe","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.991042996Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.991055476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230,"session_id":"67447857-40cc-4dc5-8c33-1b405805f4fe"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:06.991067926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 37 len 40960 data: 3 3 3 3 3 3 3 3 3 3 02/30 Write block 444 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Read block 311 len 32768 04/30 Write block 27 len 8192 data: 2 2 05/30 Read block 519 len 32768 06/30 Read block 547 len 8192 07/30 Flush 08/30 Write block 249 len 16384 data: 2 2 2 2 09/30 Read block 280 len 8192 10/30 Write block 297 len 12288 data: 3 2 2 11/30 Write block 236 len 40960 data: 2 2 2 2 2 2 2 2 2 2 12/30 Read block 561 len 12288 13/30 Flush 14/30 Write block 20 len 4096 data: 3 15/30 Read block 122 len 28672 16/30 Flush 17/30 Write block 93 len 12288 data: 2 2 3 18/30 Read block 333 len 24576 19/30 Write block 513 len 8192 data: 2 2 20/30 Write block 514 len 12288 data: 3 2 2 21/30 Read block 243 len 8192 22/30 Read block 182 len 36864 23/30 Read block 526 len 24576 24/30 Read block 181 len 36864 25/30 Write block 589 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Flush 27/30 Write block 469 len 16384 data: 3 3 3 2 28/30 Write block 326 len 20480 data: 3 3 3 2 2 29/30 Write block 306 len 8192 data: 2 2 30/30 Read block 421 len 8192 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:07.009571629Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 21 21 21 F F F 1 020-039 12 12 12 24 24 24 F F F 2 040-059 12 12 12 22 22 22 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 11 11 11 21 21 21 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 12 12 12 23 23 23 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 7 7 7 15 15 15 F F F 14 280-299 12 12 12 23 23 23 F F F 15 300-319 9 9 9 19 19 19 T T T 16 320-339 10 10 10 20 20 20 T T T 17 340-359 10 10 10 20 20 20 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 11 11 11 21 21 21 F F F 21 420-439 8 8 8 17 17 17 F F F 22 440-459 12 12 12 22 22 22 F F F 23 460-479 11 11 11 21 21 21 T T T 24 480-499 4 4 4 8 8 8 F F F 25 500-519 12 12 12 25 25 25 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 12 12 12 25 25 25 F F F Max gen: 12, Max flush: 25 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 21 21 21 F F F 1 020-039 12 12 12 24 24 24 F F F 2 040-059 12 12 12 22 22 22 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 11 11 11 21 21 21 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 11 11 11 21 21 21 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 12 12 12 23 23 23 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 7 7 7 15 15 15 F F F 14 280-299 12 12 12 23 23 23 F F F 15 300-319 13 13 13 26 26 26 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 11 11 11 21 21 21 F F F 21 420-439 8 8 8 17 17 17 F F F 22 440-459 12 12 12 22 22 22 F F F 23 460-479 13 13 13 26 26 26 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 12 12 12 25 25 25 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 7 7 7 15 15 15 F F F 29 580-599 12 12 12 25 25 25 F F F Max gen: 13, Max flush: 26 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.28882272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.331025859Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.331586662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.331621503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.331636154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.331650724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238} {"msg":"Crucible 3f851d19-cb16-4a09-9426-94543b23a7c6 has session id: e9d283b9-73f6-4c37-b828-c80d02f7a26d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.332044673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Upstairs opts: Upstairs UUID: 3f851d19-cb16-4a09-9426-94543b23a7c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.332087854Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Crucible stats registered with UUID: 3f851d19-cb16-4a09-9426-94543b23a7c6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.332107925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362636385Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362671985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238} {"msg":"3f851d19-cb16-4a09-9426-94543b23a7c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362707436Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362726037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362755018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362772858Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362790068Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362811539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362829319Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.36285148Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362885931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362920651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61908)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362956092Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_connection connected from Ok(127.0.0.1:32956)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.362989123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44557)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.363193698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.363646339Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.36367606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.36369208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.36370653Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.363737161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.363765972Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.364331785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.364362476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.364410927Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.364882918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.364908269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.364923109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.364988091Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365007241Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365021022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365033262Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365046612Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[0]R flush_numbers[0..12]: [21, 24, 22, 21, 25, 21, 20, 21, 21, 18, 1, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365064273Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[0]R generation[0..12]: [11, 12, 12, 11, 12, 11, 10, 11, 11, 8, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365078053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365092353Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[1]R flush_numbers[0..12]: [21, 24, 22, 21, 25, 21, 20, 21, 21, 18, 1, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365106244Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[1]R generation[0..12]: [11, 12, 12, 11, 12, 11, 10, 11, 11, 8, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365119954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365137684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[2]R flush_numbers[0..12]: [21, 24, 22, 21, 25, 21, 20, 21, 21, 18, 1, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365151855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[2]R generation[0..12]: [11, 12, 12, 11, 12, 11, 10, 11, 11, 8, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365165445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365179605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365192846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365206166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365219266Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365232447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"":"downstairs","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365245717Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365266047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"0","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365279578Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"1","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365293798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"client":"2","":"downstairs","session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365307388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365319819Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"3f851d19-cb16-4a09-9426-94543b23a7c6 is now active with session: e9d283b9-73f6-4c37-b828-c80d02f7a26d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.365332749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.3653471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238,"session_id":"e9d283b9-73f6-4c37-b828-c80d02f7a26d"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:11.36535987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 16384 data: 3 3 3 3 02/30 Read block 210 len 16384 03/30 Write block 100 len 24576 data: 4 4 4 3 2 2 04/30 Write block 411 len 16384 data: 2 2 2 2 05/30 Write block 16 len 36864 data: 3 3 2 2 4 3 3 2 3 06/30 Read block 434 len 40960 07/30 Read block 499 len 8192 08/30 Write block 162 len 4096 data: 2 09/30 Flush 10/30 Write block 99 len 32768 data: 4 5 5 5 4 3 3 2 11/30 Write block 23 len 20480 data: 3 4 3 2 3 12/30 Read block 76 len 28672 13/30 Read block 123 len 32768 14/30 Write block 507 len 24576 data: 3 3 3 3 2 2 15/30 Read block 577 len 12288 16/30 Flush 17/30 Write block 23 len 20480 data: 4 5 4 3 4 18/30 Read block 327 len 32768 19/30 Read block 285 len 32768 20/30 Write block 67 len 4096 data: 6 21/30 Write block 255 len 16384 data: 3 3 3 3 22/30 Write block 352 len 40960 data: 2 2 2 2 3 2 2 2 2 2 23/30 Write block 46 len 4096 data: 4 24/30 Write block 269 len 20480 data: 3 3 3 4 4 25/30 Read block 6 len 24576 26/30 Write block 458 len 8192 data: 2 2 27/30 Read block 462 len 28672 28/30 Read block 369 len 8192 29/30 Write block 562 len 36864 data: 3 3 3 3 2 2 2 2 2 30/30 Read block 377 len 16384 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 12 14 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:11.383971665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 14 14 14 28 28 28 T T T 2 040-059 12 12 12 22 22 22 T T T 3 060-079 11 11 11 21 21 21 T T T 4 080-099 14 14 14 28 28 28 F F F 5 100-119 14 14 14 28 28 28 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 14 14 14 27 27 27 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 12 12 12 23 23 23 F F F 12 240-259 12 12 12 23 23 23 T T T 13 260-279 7 7 7 15 15 15 T T T 14 280-299 12 12 12 23 23 23 F F F 15 300-319 13 13 13 26 26 26 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 10 10 10 20 20 20 T T T 18 360-379 1 1 1 1 1 1 T T T 19 380-399 11 11 11 21 21 21 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 8 8 8 17 17 17 F F F 22 440-459 12 12 12 22 22 22 T T T 23 460-479 13 13 13 26 26 26 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 14 14 14 28 28 28 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 7 7 7 15 15 15 T T T 29 580-599 12 12 12 25 25 25 F F F Max gen: 14, Max flush: 28 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 15 15 15 29 29 29 F F F 2 040-059 15 15 15 29 29 29 F F F 3 060-079 15 15 15 29 29 29 F F F 4 080-099 14 14 14 28 28 28 F F F 5 100-119 14 14 14 28 28 28 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 14 14 14 27 27 27 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 12 12 12 23 23 23 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 15 15 15 29 29 29 F F F 14 280-299 12 12 12 23 23 23 F F F 15 300-319 13 13 13 26 26 26 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 15 15 15 29 29 29 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 8 8 8 17 17 17 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 13 13 13 26 26 26 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 14 14 14 28 28 28 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 15 15 15 29 29 29 F F F 29 580-599 12 12 12 25 25 25 F F F Max gen: 15, Max flush: 29 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.668378209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.711547732Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.712241218Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.712291759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.71231924Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.712344111Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246} {"msg":"Crucible b6469e3b-5831-4978-a701-ad1e76a96485 has session id: acbbcd35-d8cf-4636-80c5-05ea76c04b4b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.71276563Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Upstairs opts: Upstairs UUID: b6469e3b-5831-4978-a701-ad1e76a96485, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.712811552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Crucible stats registered with UUID: b6469e3b-5831-4978-a701-ad1e76a96485","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.712837752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743465184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743496995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246} {"msg":"b6469e3b-5831-4978-a701-ad1e76a96485 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743531786Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743549666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743579017Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743597518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743615028Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743631798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743656339Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74368954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74371765Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743735691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59442)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743767172Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53867)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.743997957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64712)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.744089839Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.744432818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.744463378Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.744508599Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74452632Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74454049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.744554681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745162935Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745198906Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745228657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745712898Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745744809Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745761429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74577425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74578807Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74580141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74581396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745827501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[0]R flush_numbers[0..12]: [27, 29, 29, 29, 28, 28, 20, 21, 27, 18, 1, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745841291Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[0]R generation[0..12]: [14, 15, 15, 15, 14, 14, 10, 11, 14, 8, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745855232Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745869432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[1]R flush_numbers[0..12]: [27, 29, 29, 29, 28, 28, 20, 21, 27, 18, 1, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745883242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[1]R generation[0..12]: [14, 15, 15, 15, 14, 14, 10, 11, 14, 8, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745897022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745925393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[2]R flush_numbers[0..12]: [27, 29, 29, 29, 28, 28, 20, 21, 27, 18, 1, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745939923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[2]R generation[0..12]: [14, 15, 15, 15, 14, 14, 10, 11, 14, 8, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745953804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745970084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.745983524Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746073947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746092307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746105667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"":"downstairs","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746118948Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746131098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"0","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746144439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"1","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746157709Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"client":"2","":"downstairs","session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746171149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.746183539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"b6469e3b-5831-4978-a701-ad1e76a96485 is now active with session: acbbcd35-d8cf-4636-80c5-05ea76c04b4b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74619655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.7462088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246,"session_id":"acbbcd35-d8cf-4636-80c5-05ea76c04b4b"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:15.74622142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 16384 data: 3 3 3 4 02/30 Write block 430 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 215 len 4096 04/30 Write block 102 len 40960 data: 6 5 4 4 3 2 2 2 2 2 05/30 Write block 517 len 4096 data: 3 06/30 Write block 173 len 32768 data: 4 3 3 3 3 3 3 3 07/30 Read block 341 len 24576 08/30 Read block 366 len 40960 09/30 Write block 403 len 40960 data: 4 5 2 2 2 2 2 2 3 3 10/30 Read block 453 len 40960 11/30 Flush 12/30 Write block 483 len 36864 data: 2 3 3 3 3 3 3 3 3 13/30 Write block 19 len 24576 data: 3 5 4 4 5 6 14/30 Write block 45 len 32768 data: 4 5 2 2 2 2 2 3 15/30 Read block 165 len 8192 16/30 Write block 329 len 32768 data: 3 3 2 2 2 2 2 3 17/30 Write block 541 len 28672 data: 3 3 3 2 2 2 2 18/30 Read block 454 len 20480 19/30 Write block 96 len 8192 data: 3 3 20/30 Read block 138 len 16384 21/30 Write block 57 len 40960 data: 3 3 3 3 5 5 5 5 5 6 22/30 Read block 577 len 40960 23/30 Write block 198 len 24576 data: 2 2 2 2 2 2 24/30 Write block 368 len 24576 data: 2 2 2 2 2 2 25/30 Write block 569 len 32768 data: 3 3 2 2 3 3 3 3 26/30 Write block 564 len 4096 data: 4 27/30 Read block 114 len 36864 28/30 Write block 279 len 36864 data: 2 2 2 2 2 2 2 2 2 29/30 Write block 581 len 16384 data: 4 2 2 2 30/30 Read block 435 len 16384 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 19 18 17 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:15.765544382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 T T T 1 020-039 15 15 15 29 29 29 T T T 2 040-059 15 15 15 29 29 29 T T T 3 060-079 15 15 15 29 29 29 T T T 4 080-099 14 14 14 28 28 28 T T T 5 100-119 16 16 16 30 30 30 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 16 16 16 30 30 30 T T T 10 200-219 1 1 1 1 1 1 T T T 11 220-239 12 12 12 23 23 23 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 15 15 15 29 29 29 T T T 14 280-299 12 12 12 23 23 23 T T T 15 300-319 13 13 13 26 26 26 F F F 16 320-339 13 13 13 26 26 26 T T T 17 340-359 15 15 15 29 29 29 F F F 18 360-379 15 15 15 29 29 29 T T T 19 380-399 11 11 11 21 21 21 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 13 13 13 26 26 26 F F F 24 480-499 4 4 4 8 8 8 T T T 25 500-519 16 16 16 30 30 30 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 8 8 8 17 17 17 T T T 28 560-579 15 15 15 29 29 29 T T T 29 580-599 12 12 12 25 25 25 T T T Max gen: 16, Max flush: 30 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 31 31 31 F F F 1 020-039 17 17 17 31 31 31 F F F 2 040-059 17 17 17 31 31 31 F F F 3 060-079 17 17 17 31 31 31 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 17 17 17 31 31 31 F F F 11 220-239 12 12 12 23 23 23 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 17 17 17 31 31 31 F F F 14 280-299 17 17 17 31 31 31 F F F 15 300-319 13 13 13 26 26 26 F F F 16 320-339 17 17 17 31 31 31 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 17 17 17 31 31 31 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 13 13 13 26 26 26 F F F 24 480-499 17 17 17 31 31 31 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 8 8 8 17 17 17 F F F 27 540-559 17 17 17 31 31 31 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 17, Max flush: 31 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.081684225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.124624221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.125274797Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.125312738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.125328708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.125347439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254} {"msg":"Crucible bf0b90be-00f5-4ac2-9b0c-2eb035fbeb96 has session id: 20e1e1fe-1970-4fb6-a82d-f331677d0be1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.125788869Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Upstairs opts: Upstairs UUID: bf0b90be-00f5-4ac2-9b0c-2eb035fbeb96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.12583063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Crucible stats registered with UUID: bf0b90be-00f5-4ac2-9b0c-2eb035fbeb96","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.125849431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.15551158Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.15554815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254} {"msg":"bf0b90be-00f5-4ac2-9b0c-2eb035fbeb96 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155609932Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155654923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155705824Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155744035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155821867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155893319Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155921389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.15594811Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.155973141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156025972Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58224)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156058503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44496)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156084063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_connection connected from Ok(127.0.0.1:56275)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156210566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156650137Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156690868Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156844281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156875502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156950234Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.156982685Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157380184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157422565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157645841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157906467Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157931987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157950838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157978919Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.157994709Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158020189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158169773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158189354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[0]R flush_numbers[0..12]: [31, 31, 31, 31, 31, 30, 20, 21, 30, 31, 31, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158203474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[0]R generation[0..12]: [17, 17, 17, 17, 17, 16, 10, 11, 16, 17, 17, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158217674Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158232835Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[1]R flush_numbers[0..12]: [31, 31, 31, 31, 31, 30, 20, 21, 30, 31, 31, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158246805Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[1]R generation[0..12]: [17, 17, 17, 17, 17, 16, 10, 11, 16, 17, 17, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158260685Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158274596Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[2]R flush_numbers[0..12]: [31, 31, 31, 31, 31, 30, 20, 21, 30, 31, 31, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158288346Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[2]R generation[0..12]: [17, 17, 17, 17, 17, 16, 10, 11, 16, 17, 17, 12]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158301876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158319227Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158333327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158346797Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158359978Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158373178Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158398779Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158418189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"0","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158436159Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"1","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.15846145Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"client":"2","":"downstairs","session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158489521Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158515752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"bf0b90be-00f5-4ac2-9b0c-2eb035fbeb96 is now active with session: 20e1e1fe-1970-4fb6-a82d-f331677d0be1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158543292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:20.158569483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254,"session_id":"20e1e1fe-1970-4fb6-a82d-f331677d0be1"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2024-12-09T15:08:20.158599923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Read block 130 len 12288 03/30 Read block 59 len 4096 04/30 Write block 303 len 36864 data: 2 2 2 3 3 2 2 2 2 05/30 Read block 105 len 8192 06/30 Read block 90 len 36864 07/30 Read block 109 len 4096 08/30 Write block 374 len 24576 data: 2 2 2 3 3 3 09/30 Read block 391 len 8192 10/30 Read block 324 len 4096 11/30 Read block 48 len 4096 12/30 Read block 492 len 32768 13/30 Read block 533 len 24576 14/30 Read block 83 len 36864 15/30 Write block 478 len 20480 data: 2 2 2 2 2 16/30 Read block 64 len 16384 17/30 Write block 155 len 32768 data: 3 3 3 3 4 4 4 3 18/30 Read block 354 len 8192 19/30 Write block 402 len 28672 data: 4 5 6 3 3 3 3 20/30 Read block 497 len 32768 21/30 Read block 410 len 16384 22/30 Write block 479 len 12288 data: 3 3 3 23/30 Flush 24/30 Write block 119 len 12288 data: 3 2 2 25/30 Write block 536 len 36864 data: 2 3 3 3 3 4 4 4 3 26/30 Read block 186 len 20480 27/30 Read block 272 len 40960 28/30 Write block 219 len 32768 data: 2 2 2 3 3 4 5 4 29/30 Read block 490 len 24576 30/30 Read block 85 len 40960 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:20.179505633Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 31 31 31 F F F 1 020-039 17 17 17 31 31 31 F F F 2 040-059 17 17 17 31 31 31 F F F 3 060-079 17 17 17 31 31 31 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 16 16 16 30 30 30 T T T 6 120-139 10 10 10 20 20 20 T T T 7 140-159 18 18 18 32 32 32 F F F 8 160-179 18 18 18 32 32 32 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 17 17 17 31 31 31 T T T 11 220-239 12 12 12 23 23 23 T T T 12 240-259 15 15 15 29 29 29 F F F 13 260-279 17 17 17 31 31 31 F F F 14 280-299 17 17 17 31 31 31 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 17 17 17 31 31 31 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 8 8 8 17 17 17 T T T 27 540-559 17 17 17 31 31 31 T T T 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 18, Max flush: 32 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 31 31 31 F F F 1 020-039 17 17 17 31 31 31 F F F 2 040-059 17 17 17 31 31 31 F F F 3 060-079 17 17 17 31 31 31 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 19 19 19 33 33 33 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 18 18 18 32 32 32 F F F 8 160-179 18 18 18 32 32 32 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 19 19 19 33 33 33 F F F 11 220-239 19 19 19 33 33 33 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 17 17 17 31 31 31 F F F 14 280-299 17 17 17 31 31 31 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 17 17 17 31 31 31 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 19 19 19 33 33 33 F F F 27 540-559 19 19 19 33 33 33 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 19, Max flush: 33 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.444136755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.486751294Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.487394109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.48744613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.487471511Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.487497161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262} {"msg":"Crucible 0024f18a-c3a0-4908-bc9c-cbf94299d3e2 has session id: 73e56804-1ae1-4101-b0d1-b4788d3091ca","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.48787319Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Upstairs opts: Upstairs UUID: 0024f18a-c3a0-4908-bc9c-cbf94299d3e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.487917422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Crucible stats registered with UUID: 0024f18a-c3a0-4908-bc9c-cbf94299d3e2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.487937992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517467068Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517494359Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262} {"msg":"0024f18a-c3a0-4908-bc9c-cbf94299d3e2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517590861Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517609312Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517633052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517647712Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517661763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517675223Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517688763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517702194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517718964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517733004Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:37573)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.517821417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:35582)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518042152Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:36184)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518083743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518571984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518612625Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518642756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518715088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518747059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.518835351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.51921378Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519296162Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519381424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519797324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519822154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519837225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519926407Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519955548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519970578Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.519983388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520001209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[0]R flush_numbers[0..12]: [31, 31, 31, 31, 31, 33, 33, 32, 32, 31, 33, 33]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520015989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[0]R generation[0..12]: [17, 17, 17, 17, 17, 19, 19, 18, 18, 17, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520031079Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.52004641Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[1]R flush_numbers[0..12]: [31, 31, 31, 31, 31, 33, 33, 32, 32, 31, 33, 33]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.52006062Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[1]R generation[0..12]: [17, 17, 17, 17, 17, 19, 19, 18, 18, 17, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.5200744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520088411Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[2]R flush_numbers[0..12]: [31, 31, 31, 31, 31, 33, 33, 32, 32, 31, 33, 33]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520103201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[2]R generation[0..12]: [17, 17, 17, 17, 17, 19, 19, 18, 18, 17, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520117241Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520131372Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520144792Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520167793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520192443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520213804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"":"downstairs","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520230784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520244674Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"0","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520267825Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"1","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520289316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"client":"2","":"downstairs","session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520303546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520317066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"0024f18a-c3a0-4908-bc9c-cbf94299d3e2 is now active with session: 73e56804-1ae1-4101-b0d1-b4788d3091ca","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520332606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520355717Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262,"session_id":"73e56804-1ae1-4101-b0d1-b4788d3091ca"} {"msg":"The guest has finished waiting for activation with:20","vDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:24.520375938Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 36864 data: 2 5 4 4 4 4 4 4 4 02/30 Read block 43 len 20480 03/30 Flush 04/30 Write block 20 len 12288 data: 6 5 5 05/30 Read block 422 len 12288 06/30 Read block 266 len 32768 07/30 Read block 115 len 8192 08/30 Read block 219 len 32768 09/30 Write block 207 len 12288 data: 2 2 2 10/30 Write block 287 len 32768 data: 3 2 2 3 3 3 3 3 11/30 Read block 486 len 40960 12/30 Write block 232 len 8192 data: 2 2 13/30 Read block 206 len 12288 14/30 Read block 252 len 36864 15/30 Read block 335 len 12288 16/30 Read block 393 len 4096 17/30 Write block 250 len 40960 data: 3 3 3 2 2 4 4 4 4 3 18/30 Write block 544 len 16384 data: 4 3 3 3 19/30 Read block 556 len 16384 20/30 Flush 21/30 Flush 22/30 Write block 362 len 28672 data: 2 2 2 2 2 2 3 23/30 Write block 281 len 40960 data: 3 3 3 3 3 3 4 3 3 4 24/30 Read block 221 len 16384 25/30 Read block 99 len 36864 26/30 Read block 441 len 4096 27/30 Read block 531 len 4096 28/30 Read block 21 len 20480 29/30 Write block 524 len 36864 data: 3 3 2 2 2 2 2 2 2 30/30 Write block 550 len 20480 data: 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:24.540220632Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 31 31 31 F F F 1 020-039 20 20 20 35 35 35 F F F 2 040-059 17 17 17 31 31 31 F F F 3 060-079 17 17 17 31 31 31 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 19 19 19 33 33 33 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 18 18 18 32 32 32 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 20 20 20 35 35 35 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 17 17 17 31 31 31 F F F 14 280-299 20 20 20 35 35 35 T T T 15 300-319 18 18 18 32 32 32 F F F 16 320-339 17 17 17 31 31 31 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 18 18 18 32 32 32 T T T 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 19 19 19 33 33 33 T T T 27 540-559 20 20 20 35 35 35 T T T 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 20, Max flush: 35 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 31 31 31 F F F 1 020-039 20 20 20 35 35 35 F F F 2 040-059 17 17 17 31 31 31 F F F 3 060-079 17 17 17 31 31 31 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 19 19 19 33 33 33 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 18 18 18 32 32 32 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 20 20 20 35 35 35 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 17 17 17 31 31 31 F F F 14 280-299 21 21 21 36 36 36 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 17 17 17 31 31 31 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 21 21 21 36 36 36 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 21 21 21 36 36 36 F F F 27 540-559 21 21 21 36 36 36 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 21, Max flush: 36 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.792973449Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:28.83441569Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.834774288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.865648836Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.86620451Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.866240681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.866255471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.866270021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270} {"msg":"Crucible 1dd42170-cf66-4a34-ae52-c688039894c2 has session id: 4588cfd6-b95c-4775-86be-a805ec0a276f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.86665237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Upstairs opts: Upstairs UUID: 1dd42170-cf66-4a34-ae52-c688039894c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.866701421Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Crucible stats registered with UUID: 1dd42170-cf66-4a34-ae52-c688039894c2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.866728972Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.896895293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.896928424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270} {"msg":"1dd42170-cf66-4a34-ae52-c688039894c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.896963875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.896982925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897011516Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897029266Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897046527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897063207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897080148Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897096798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897119549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897153349Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44067)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897419576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55846)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897462287Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43558)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.897495408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898066161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898096362Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898112242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898143143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898174124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898201225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898759218Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898793869Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.898825669Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.89926457Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.89929048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899304921Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899379183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899398113Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899411723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899423904Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899437464Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[0]R flush_numbers[0..12]: [31, 35, 31, 31, 31, 33, 33, 32, 34, 34, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899451554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[0]R generation[0..12]: [17, 20, 17, 17, 17, 19, 19, 18, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899469395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899484445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[1]R flush_numbers[0..12]: [31, 35, 31, 31, 31, 33, 33, 32, 34, 34, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899498405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[1]R generation[0..12]: [17, 20, 17, 17, 17, 19, 19, 18, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899511876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899526046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[2]R flush_numbers[0..12]: [31, 35, 31, 31, 31, 33, 33, 32, 34, 34, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899539906Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[2]R generation[0..12]: [17, 20, 17, 17, 17, 19, 19, 18, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899553627Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899568277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899581687Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899594998Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","Wait for a query_work_queue command to finish before sending IO pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.89967838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.89969683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"":"downstairs","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899710661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899722941Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"0","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899738921Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"1","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899753111Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"client":"2","":"downstairs","session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899766912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899779262Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"1dd42170-cf66-4a34-ae52-c688039894c2 is now active with session: 4588cfd6-b95c-4775-86be-a805ec0a276f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899792263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899804723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270,"session_id":"4588cfd6-b95c-4775-86be-a805ec0a276f"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:28.899817243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 20480 data: 3 3 3 3 3 02/30 Read block 152 len 40960 03/30 Read block 73 len 36864 04/30 Read block 122 len 8192 05/30 Read block 592 len 20480 06/30 Read block 473 len 12288 07/30 Write block 478 len 40960 data: 3 4 4 4 3 3 4 4 4 4 08/30 Flush 09/30 Write block 278 len 12288 data: 2 3 3 10/30 Read block 231 len 12288 11/30 Flush 12/30 Write block 314 len 8192 data: 4 4 13/30 Read block 372 len 28672 14/30 Read block 296 len 36864 15/30 Write block 273 len 36864 data: 5 4 4 2 2 3 4 4 4 16/30 Write block 313 len 4096 data: 3 17/30 Write block 534 len 16384 data: 2 2 3 4 18/30 Write block 515 len 4096 data: 3 19/30 Flush 20/30 Write block 156 len 16384 data: 4 4 4 5 21/30 Read block 356 len 4096 22/30 Flush 23/30 Write block 554 len 16384 data: 4 2 2 2 24/30 Write block 49 len 36864 data: 3 3 3 4 3 3 3 3 4 25/30 Read block 535 len 28672 26/30 Write block 227 len 28672 data: 4 3 3 2 2 3 3 27/30 Read block 133 len 24576 28/30 Read block 215 len 24576 29/30 Write block 52 len 20480 data: 5 4 4 4 4 30/30 Read block 431 len 24576 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:28.918680784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 31 31 31 F F F 1 020-039 20 20 20 35 35 35 F F F 2 040-059 17 17 17 31 31 31 T T T 3 060-079 17 17 17 31 31 31 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 19 19 19 33 33 33 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 20 20 20 35 35 35 T T T 12 240-259 20 20 20 35 35 35 F F F 13 260-279 22 22 22 39 39 39 F F F 14 280-299 22 22 22 39 39 39 F F F 15 300-319 22 22 22 39 39 39 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 21 21 21 36 36 36 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 22 22 22 37 37 37 F F F 24 480-499 22 22 22 37 37 37 F F F 25 500-519 22 22 22 39 39 39 F F F 26 520-539 22 22 22 39 39 39 F F F 27 540-559 21 21 21 36 36 36 T T T 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 31 31 31 F F F 1 020-039 20 20 20 35 35 35 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 17 17 17 31 31 31 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 19 19 19 33 33 33 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 22 22 22 39 39 39 F F F 14 280-299 22 22 22 39 39 39 F F F 15 300-319 22 22 22 39 39 39 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 21 21 21 36 36 36 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 22 22 22 37 37 37 F F F 24 480-499 22 22 22 37 37 37 F F F 25 500-519 22 22 22 39 39 39 F F F 26 520-539 22 22 22 39 39 39 F F F 27 540-559 23 23 23 41 41 41 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.198898789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:33.240171935Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.240528074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.271063544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.271637937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.271673888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.271689018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.271703559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278} {"msg":"Crucible da663b6e-235a-4e9e-81d2-c7d5bb42a842 has session id: 883b304f-2d23-4a67-9bfb-0895b076bd10","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.272147209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Upstairs opts: Upstairs UUID: da663b6e-235a-4e9e-81d2-c7d5bb42a842, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.27218175Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Crucible stats registered with UUID: da663b6e-235a-4e9e-81d2-c7d5bb42a842","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.272198121Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302639109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302665839Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278} {"msg":"da663b6e-235a-4e9e-81d2-c7d5bb42a842 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302746691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302769361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302800032Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302822823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302840883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302920605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.302971396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303172271Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303212452Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303234513Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_connection connected from Ok(127.0.0.1:64344)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303253903Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_connection connected from Ok(127.0.0.1:61664)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303272273Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_connection connected from Ok(127.0.0.1:38576)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303296324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303811726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303848557Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303877288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303906189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.303932789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.30396371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.304408701Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.304593055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.304700798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.304989195Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305047136Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305075417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305100797Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305124928Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305140218Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305153038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305166549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[0]R flush_numbers[0..12]: [31, 35, 41, 31, 31, 33, 33, 40, 34, 34, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.30519165Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[0]R generation[0..12]: [17, 20, 23, 17, 17, 19, 19, 22, 20, 20, 20, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.30520648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.30522085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[1]R flush_numbers[0..12]: [31, 35, 41, 31, 31, 33, 33, 40, 34, 34, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305237931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[1]R generation[0..12]: [17, 20, 23, 17, 17, 19, 19, 22, 20, 20, 20, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305252211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305266451Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[2]R flush_numbers[0..12]: [31, 35, 41, 31, 31, 33, 33, 40, 34, 34, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305280032Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[2]R generation[0..12]: [17, 20, 23, 17, 17, 19, 19, 22, 20, 20, 20, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305293742Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305307862Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305321202Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305334443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305347473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-12-09T15:08:33.305360443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"":"downstairs","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305381684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305393924Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"0","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305407415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"1","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305420595Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"client":"2","":"downstairs","session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305433845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305448506Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"da663b6e-235a-4e9e-81d2-c7d5bb42a842 is now active with session: 883b304f-2d23-4a67-9bfb-0895b076bd10","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305461576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305473666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278,"session_id":"883b304f-2d23-4a67-9bfb-0895b076bd10"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:33.305486077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 24576 data: 4 4 3 4 7 6 02/30 Read block 529 len 4096 03/30 Read block 100 len 20480 04/30 Read block 203 len 36864 05/30 Write block 304 len 20480 data: 3 3 4 4 3 06/30 Write block 424 len 12288 data: 3 3 3 07/30 Write block 476 len 8192 data: 2 2 08/30 Read block 17 len 20480 09/30 Write block 136 len 24576 data: 4 2 2 2 3 4 10/30 Write block 377 len 8192 data: 4 4 11/30 Write block 158 len 20480 data: 5 6 5 5 4 12/30 Flush 13/30 Write block 266 len 8192 data: 2 3 14/30 Read block 277 len 36864 15/30 Flush 16/30 Write block 551 len 32768 data: 3 3 4 5 3 3 3 2 17/30 Read block 390 len 40960 18/30 Write block 138 len 40960 data: 3 3 4 5 4 4 4 3 3 3 19/30 Read block 567 len 24576 20/30 Flush 21/30 Write block 98 len 36864 data: 4 5 6 6 7 6 5 5 4 22/30 Read block 169 len 12288 23/30 Write block 194 len 20480 data: 2 2 2 3 3 24/30 Flush 25/30 Write block 510 len 36864 data: 4 3 3 3 4 4 3 4 3 26/30 Write block 463 len 4096 data: 3 27/30 Write block 425 len 8192 data: 4 4 28/30 Write block 77 len 28672 data: 3 3 3 2 2 3 2 29/30 Read block 397 len 28672 30/30 Read block 537 len 8192 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:33.324774878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 17 17 17 31 31 31 T T T 4 080-099 24 24 24 45 45 45 T T T 5 100-119 24 24 24 45 45 45 F F F 6 120-139 24 24 24 44 44 44 F F F 7 140-159 24 24 24 44 44 44 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 24 24 24 43 43 43 F F F 14 280-299 22 22 22 39 39 39 F F F 15 300-319 24 24 24 42 42 42 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 24 24 24 42 42 42 T T T 22 440-459 15 15 15 29 29 29 F F F 23 460-479 24 24 24 42 42 42 T T T 24 480-499 22 22 22 37 37 37 F F F 25 500-519 22 22 22 39 39 39 T T T 26 520-539 22 22 22 39 39 39 F F F 27 540-559 24 24 24 44 44 44 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 24, Max flush: 45 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 25 25 25 46 46 46 F F F 4 080-099 25 25 25 46 46 46 F F F 5 100-119 24 24 24 45 45 45 F F F 6 120-139 24 24 24 44 44 44 F F F 7 140-159 24 24 24 44 44 44 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 24 24 24 43 43 43 F F F 14 280-299 22 22 22 39 39 39 F F F 15 300-319 24 24 24 42 42 42 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 25 25 25 46 46 46 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 25 25 25 46 46 46 F F F 24 480-499 22 22 22 37 37 37 F F F 25 500-519 25 25 25 46 46 46 F F F 26 520-539 22 22 22 39 39 39 F F F 27 540-559 24 24 24 44 44 44 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 17 17 17 31 31 31 F F F Max gen: 25, Max flush: 46 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.6128609Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.655424878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.656041212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.656077413Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.656097184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.656112554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286} {"msg":"Crucible 6d87eff9-11b3-4212-b972-f87ee2de951e has session id: f67febaa-30fe-4df6-8796-9838b12cc6ae","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.656548254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Upstairs opts: Upstairs UUID: 6d87eff9-11b3-4212-b972-f87ee2de951e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.656594856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Crucible stats registered with UUID: 6d87eff9-11b3-4212-b972-f87ee2de951e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.656620936Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686380718Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686425669Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286} {"msg":"6d87eff9-11b3-4212-b972-f87ee2de951e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.68649094Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686519711Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686545282Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686560342Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686574682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686588473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686602503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686620123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686634744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686648554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:65364)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686784857Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:49084)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.686830728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:36017)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.68690543Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.687504945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.687534735Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.687550066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.687564336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.687578106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.687592607Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688103369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688258892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688301224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.6885777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688608901Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688626501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688793435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688818516Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688834276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688859537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688873977Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[0]R flush_numbers[0..12]: [42, 42, 41, 46, 46, 45, 44, 44, 42, 45, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688912338Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[0]R generation[0..12]: [24, 24, 23, 25, 25, 24, 24, 24, 24, 24, 20, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688929379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688943789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[1]R flush_numbers[0..12]: [42, 42, 41, 46, 46, 45, 44, 44, 42, 45, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688957799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[1]R generation[0..12]: [24, 24, 23, 25, 25, 24, 24, 24, 24, 24, 20, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.688971729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.68900348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[2]R flush_numbers[0..12]: [42, 42, 41, 46, 46, 45, 44, 44, 42, 45, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689019621Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[2]R generation[0..12]: [24, 24, 23, 25, 25, 24, 24, 24, 24, 24, 20, 23]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689037121Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689051802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689064912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689078092Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689091032Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689113063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"":"downstairs","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689126343Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689138504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"0","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689151804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"1","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689164984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"client":"2","":"downstairs","session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689178245Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689190605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"6d87eff9-11b3-4212-b972-f87ee2de951e is now active with session: f67febaa-30fe-4df6-8796-9838b12cc6ae","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689203535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:08:37.689215765Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286,"session_id":"f67febaa-30fe-4df6-8796-9838b12cc6ae"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:37.689231336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 500 len 4096 data: 2 02/30 Flush 03/30 Write block 41 len 40960 data: 4 4 4 4 5 6 3 3 4 4 04/30 Read block 481 len 16384 05/30 Write block 237 len 12288 data: 3 3 3 06/30 Read block 349 len 20480 07/30 Read block 88 len 28672 08/30 Write block 154 len 16384 data: 3 4 5 5 09/30 Write block 300 len 36864 data: 2 3 3 3 4 4 5 5 4 10/30 Read block 281 len 24576 11/30 Read block 255 len 8192 12/30 Read block 333 len 28672 13/30 Read block 512 len 36864 14/30 Read block 468 len 12288 15/30 Write block 543 len 4096 data: 5 16/30 Read block 569 len 20480 17/30 Write block 583 len 16384 data: 3 3 2 2 18/30 Read block 464 len 36864 19/30 Write block 481 len 12288 data: 5 4 4 20/30 Read block 572 len 4096 21/30 Flush 22/30 Write block 286 len 28672 data: 4 5 4 4 5 4 4 23/30 Write block 77 len 8192 data: 4 4 24/30 Read block 512 len 24576 25/30 Read block 500 len 12288 26/30 Read block 147 len 36864 27/30 Write block 456 len 36864 data: 2 2 3 3 2 2 3 4 3 28/30 Write block 346 len 24576 data: 2 2 2 2 2 2 29/30 Read block 39 len 8192 30/30 Read block 224 len 40960 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:37.708718672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 26 26 26 48 48 48 F F F 3 060-079 25 25 25 46 46 46 T T T 4 080-099 25 25 25 46 46 46 F F F 5 100-119 24 24 24 45 45 45 F F F 6 120-139 24 24 24 44 44 44 F F F 7 140-159 26 26 26 48 48 48 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 26 26 26 48 48 48 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 24 24 24 43 43 43 F F F 14 280-299 22 22 22 39 39 39 T T T 15 300-319 26 26 26 48 48 48 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 15 15 15 29 29 29 T T T 18 360-379 24 24 24 42 42 42 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 25 25 25 46 46 46 F F F 22 440-459 15 15 15 29 29 29 T T T 23 460-479 25 25 25 46 46 46 T T T 24 480-499 26 26 26 48 48 48 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 22 22 22 39 39 39 F F F 27 540-559 26 26 26 48 48 48 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 26 26 26 48 48 48 F F F Max gen: 26, Max flush: 48 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 26 26 26 48 48 48 F F F 3 060-079 27 27 27 49 49 49 F F F 4 080-099 25 25 25 46 46 46 F F F 5 100-119 24 24 24 45 45 45 F F F 6 120-139 24 24 24 44 44 44 F F F 7 140-159 26 26 26 48 48 48 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 26 26 26 48 48 48 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 24 24 24 43 43 43 F F F 14 280-299 27 27 27 49 49 49 F F F 15 300-319 26 26 26 48 48 48 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 27 27 27 49 49 49 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 25 25 25 46 46 46 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 27 27 27 49 49 49 F F F 24 480-499 26 26 26 48 48 48 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 22 22 22 39 39 39 F F F 27 540-559 26 26 26 48 48 48 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 26 26 26 48 48 48 F F F Max gen: 27, Max flush: 49 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:41.96569689Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.008181366Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.008842862Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.008879852Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.008894653Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.008909613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294} {"msg":"Crucible 011e09b9-3d7a-4ef5-a03d-7be4fcd9ef71 has session id: 2da4526c-620c-40ef-9188-6f260125c0de","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.009324473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Upstairs opts: Upstairs UUID: 011e09b9-3d7a-4ef5-a03d-7be4fcd9ef71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.009364724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Crucible stats registered with UUID: 011e09b9-3d7a-4ef5-a03d-7be4fcd9ef71","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.009388925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039296039Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.03933196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294} {"msg":"011e09b9-3d7a-4ef5-a03d-7be4fcd9ef71 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039360331Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039384892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039434303Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039473194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039510745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039544036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039576686Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039612167Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039649558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039686179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_connection connected from Ok(127.0.0.1:49503)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.03971985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_connection connected from Ok(127.0.0.1:47628)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039759581Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_connection connected from Ok(127.0.0.1:42241)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.039793691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.040340824Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.040376735Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.040394756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.040452217Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.040474248Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.040488738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.040949199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041175444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041207695Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041489462Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041519743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041539173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041719957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041746748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041764769Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041783799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041798149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[0]R flush_numbers[0..12]: [42, 42, 48, 49, 46, 45, 44, 48, 42, 45, 35, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.04183764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[0]R generation[0..12]: [24, 24, 26, 27, 25, 24, 24, 26, 24, 24, 20, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041855281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041869791Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[1]R flush_numbers[0..12]: [42, 42, 48, 49, 46, 45, 44, 48, 42, 45, 35, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041883691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[1]R generation[0..12]: [24, 24, 26, 27, 25, 24, 24, 26, 24, 24, 20, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041897522Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041911682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[2]R flush_numbers[0..12]: [42, 42, 48, 49, 46, 45, 44, 48, 42, 45, 35, 48]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041925502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[2]R generation[0..12]: [24, 24, 26, 27, 25, 24, 24, 26, 24, 24, 20, 26]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041939173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041953223Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.041967283Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2024-12-09T15:08:42.041991154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042014384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042039825Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"":"downstairs","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042067426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042091786Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"0","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042106947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"1","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042125457Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"client":"2","":"downstairs","session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042151398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042177448Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"011e09b9-3d7a-4ef5-a03d-7be4fcd9ef71 is now active with session: 2da4526c-620c-40ef-9188-6f260125c0de","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042198519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.042211529Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294,"session_id":"2da4526c-620c-40ef-9188-6f260125c0de"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:42.04223187Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 4096 data: 2 02/30 Read block 199 len 12288 03/30 Read block 416 len 4096 04/30 Write block 580 len 16384 data: 4 5 3 4 05/30 Read block 131 len 40960 06/30 Write block 552 len 24576 data: 4 5 6 4 4 4 07/30 Read block 299 len 20480 08/30 Write block 369 len 24576 data: 3 3 3 3 3 3 09/30 Write block 377 len 24576 data: 5 5 4 3 3 3 10/30 Write block 118 len 20480 data: 3 4 3 3 3 11/30 Write block 532 len 16384 data: 3 2 3 3 12/30 Read block 291 len 4096 13/30 Read block 541 len 40960 14/30 Flush 15/30 Write block 434 len 20480 data: 3 3 3 2 2 16/30 Write block 185 len 12288 data: 2 2 2 17/30 Write block 234 len 4096 data: 2 18/30 Read block 20 len 40960 19/30 Read block 103 len 24576 20/30 Read block 592 len 4096 21/30 Write block 38 len 24576 data: 4 4 4 5 5 5 22/30 Write block 220 len 32768 data: 3 3 4 4 5 6 5 5 23/30 Read block 519 len 24576 24/30 Read block 80 len 32768 25/30 Write block 141 len 4096 data: 6 26/30 Write block 238 len 28672 data: 4 4 3 3 3 3 3 27/30 Write block 363 len 4096 data: 3 28/30 Read block 208 len 36864 29/30 Write block 337 len 36864 data: 3 4 4 4 4 4 4 4 2 30/30 Write block 530 len 36864 data: 3 3 4 3 4 4 4 5 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:42.059709518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 24 24 24 42 42 42 T T T 2 040-059 26 26 26 48 48 48 T T T 3 060-079 27 27 27 49 49 49 F F F 4 080-099 25 25 25 46 46 46 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 28 28 28 50 50 50 F F F 7 140-159 26 26 26 48 48 48 T T T 8 160-179 28 28 28 50 50 50 F F F 9 180-199 24 24 24 45 45 45 T T T 10 200-219 20 20 20 35 35 35 F F F 11 220-239 26 26 26 48 48 48 T T T 12 240-259 20 20 20 35 35 35 T T T 13 260-279 24 24 24 43 43 43 F F F 14 280-299 27 27 27 49 49 49 F F F 15 300-319 26 26 26 48 48 48 F F F 16 320-339 22 22 22 37 37 37 T T T 17 340-359 27 27 27 49 49 49 T T T 18 360-379 28 28 28 50 50 50 T T T 19 380-399 28 28 28 50 50 50 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 25 25 25 46 46 46 T T T 22 440-459 27 27 27 49 49 49 F F F 23 460-479 27 27 27 49 49 49 F F F 24 480-499 26 26 26 48 48 48 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 28 28 28 50 50 50 T T T 27 540-559 28 28 28 50 50 50 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 28, Max flush: 50 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 29 29 29 51 51 51 F F F 2 040-059 29 29 29 51 51 51 F F F 3 060-079 27 27 27 49 49 49 F F F 4 080-099 25 25 25 46 46 46 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 28 28 28 50 50 50 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 29 29 29 51 51 51 F F F 13 260-279 24 24 24 43 43 43 F F F 14 280-299 27 27 27 49 49 49 F F F 15 300-319 26 26 26 48 48 48 F F F 16 320-339 29 29 29 51 51 51 F F F 17 340-359 29 29 29 51 51 51 F F F 18 360-379 29 29 29 51 51 51 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 27 27 27 49 49 49 F F F 24 480-499 26 26 26 48 48 48 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 29 29 29 51 51 51 F F F 27 540-559 28 28 28 50 50 50 F F F 28 560-579 17 17 17 31 31 31 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 29, Max flush: 51 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.347217296Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.389681981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.390323986Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.390359937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.390374558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.390388958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302} {"msg":"Crucible eb668823-c880-438b-89f3-11ddcf5d626c has session id: 5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.390767227Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Upstairs opts: Upstairs UUID: eb668823-c880-438b-89f3-11ddcf5d626c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.390814388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Crucible stats registered with UUID: eb668823-c880-438b-89f3-11ddcf5d626c","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.390843889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422459115Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422505056Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302} {"msg":"eb668823-c880-438b-89f3-11ddcf5d626c active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422573468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422601958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422627009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422642059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.42265645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.42267036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.42268394Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.42269787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422712071Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422726101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45608)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422921386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61048)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.422972417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57673)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.42311606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.423779196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.423807517Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.423823247Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.423837038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.423850368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.423863568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.424518684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.424549755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.424568295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425107228Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425137089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425153339Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425215731Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425236341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425250561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425263052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425280392Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[0]R flush_numbers[0..12]: [42, 51, 51, 49, 46, 50, 50, 51, 50, 51, 35, 51]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425295462Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[0]R generation[0..12]: [24, 29, 29, 27, 25, 28, 28, 29, 28, 29, 20, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425309503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425323893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[1]R flush_numbers[0..12]: [42, 51, 51, 49, 46, 50, 50, 51, 50, 51, 35, 51]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425337803Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[1]R generation[0..12]: [24, 29, 29, 27, 25, 28, 28, 29, 28, 29, 20, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425351454Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425365484Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[2]R flush_numbers[0..12]: [42, 51, 51, 49, 46, 50, 50, 51, 50, 51, 35, 51]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425379084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[2]R generation[0..12]: [24, 29, 29, 27, 25, 28, 28, 29, 28, 29, 20, 29]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425392655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425406755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425420185Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425433746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425447006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"Wait for a query_work_queue command to finish before sending IO msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425471237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"":"downstairs","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425486897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425499527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"0","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425512958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"1","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425526348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"client":"2","":"downstairs","session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425539708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425552189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"eb668823-c880-438b-89f3-11ddcf5d626c is now active with session: 5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425565149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425577349Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302,"session_id":"5e6fa3a5-b2c2-4e8d-9416-b4b595443c2c"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:46.425589889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 8192 data: 3 4 02/30 Read block 36 len 12288 03/30 Read block 126 len 12288 04/30 Read block 2 len 16384 05/30 Read block 402 len 32768 06/30 Write block 234 len 8192 data: 3 2 07/30 Read block 145 len 16384 08/30 Write block 540 len 4096 data: 4 09/30 Read block 566 len 32768 10/30 Flush 11/30 Write block 36 len 20480 data: 2 4 5 5 5 12/30 Read block 436 len 4096 13/30 Read block 181 len 28672 14/30 Read block 501 len 20480 15/30 Read block 99 len 8192 16/30 Write block 574 len 24576 data: 4 4 4 3 3 4 17/30 Read block 91 len 20480 18/30 Read block 476 len 16384 19/30 Write block 181 len 4096 data: 2 20/30 Read block 190 len 20480 21/30 Read block 184 len 12288 22/30 Write block 67 len 32768 data: 7 4 4 4 3 3 3 3 23/30 Read block 408 len 20480 24/30 Read block 266 len 28672 25/30 Read block 376 len 4096 26/30 Read block 283 len 20480 27/30 Read block 458 len 24576 28/30 Read block 16 len 4096 29/30 Write block 4 len 28672 data: 3 2 2 2 2 2 2 30/30 Write block 337 len 20480 data: 4 5 5 5 5 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 5 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:46.444375509Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 T T T 1 020-039 29 29 29 51 51 51 T T T 2 040-059 29 29 29 51 51 51 T T T 3 060-079 27 27 27 49 49 49 T T T 4 080-099 25 25 25 46 46 46 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 28 28 28 50 50 50 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 29 29 29 51 51 51 T T T 10 200-219 20 20 20 35 35 35 F F F 11 220-239 30 30 30 52 52 52 F F F 12 240-259 29 29 29 51 51 51 F F F 13 260-279 30 30 30 52 52 52 F F F 14 280-299 27 27 27 49 49 49 F F F 15 300-319 26 26 26 48 48 48 F F F 16 320-339 29 29 29 51 51 51 T T T 17 340-359 29 29 29 51 51 51 T T T 18 360-379 29 29 29 51 51 51 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 27 27 27 49 49 49 F F F 24 480-499 26 26 26 48 48 48 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 29 29 29 51 51 51 F F F 27 540-559 30 30 30 52 52 52 F F F 28 560-579 17 17 17 31 31 31 T T T 29 580-599 28 28 28 50 50 50 F F F Max gen: 30, Max flush: 52 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 53 53 53 F F F 1 020-039 31 31 31 53 53 53 F F F 2 040-059 31 31 31 53 53 53 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 25 25 25 46 46 46 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 28 28 28 50 50 50 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 30 30 30 52 52 52 F F F 12 240-259 29 29 29 51 51 51 F F F 13 260-279 30 30 30 52 52 52 F F F 14 280-299 27 27 27 49 49 49 F F F 15 300-319 26 26 26 48 48 48 F F F 16 320-339 31 31 31 53 53 53 F F F 17 340-359 31 31 31 53 53 53 F F F 18 360-379 29 29 29 51 51 51 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 27 27 27 49 49 49 F F F 24 480-499 26 26 26 48 48 48 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 29 29 29 51 51 51 F F F 27 540-559 30 30 30 52 52 52 F F F 28 560-579 31 31 31 53 53 53 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 31, Max flush: 53 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.720805362Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:50.761819053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.762191252Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.792629479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.793190292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.793225813Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.793239944Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.793254344Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310} {"msg":"Crucible b5cbe8d9-70ca-4eba-817f-8598d947e842 has session id: 4d5e62ed-927c-4701-bfb4-a02cc90610cf","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.793655793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Upstairs opts: Upstairs UUID: b5cbe8d9-70ca-4eba-817f-8598d947e842, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.793702224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Crucible stats registered with UUID: b5cbe8d9-70ca-4eba-817f-8598d947e842","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.793727075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.82403194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.82406399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310} {"msg":"b5cbe8d9-70ca-4eba-817f-8598d947e842 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824121222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824149512Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824175473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824190483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824215424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824242174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824270435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824295906Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824311246Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824325147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:62082)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.82447357Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:60081)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824510541Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:59970)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.824578622Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825194137Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825224918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825379532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825409882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825426253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825440403Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825915314Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.825946665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826209881Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826477298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826503299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826519299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826583801Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826626382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826655712Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826683613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826712073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[0]R flush_numbers[0..12]: [53, 53, 53, 53, 46, 50, 50, 51, 50, 53, 35, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826737724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[0]R generation[0..12]: [31, 31, 31, 31, 25, 28, 28, 29, 28, 31, 20, 30]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826763415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826790055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[1]R flush_numbers[0..12]: [53, 53, 53, 53, 46, 50, 50, 51, 50, 53, 35, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826814746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[1]R generation[0..12]: [31, 31, 31, 31, 25, 28, 28, 29, 28, 31, 20, 30]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826838417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826865417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[2]R flush_numbers[0..12]: [53, 53, 53, 53, 46, 50, 50, 51, 50, 53, 35, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826892758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[2]R generation[0..12]: [31, 31, 31, 31, 25, 28, 28, 29, 28, 31, 20, 30]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826918749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.826946389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Max found gen is 32","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-12-09T15:08:50.82698306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827033431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827064922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827093283Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"":"downstairs","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827125574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827153584Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"0","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827185665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"1","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827213776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"client":"2","":"downstairs","session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827230006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827243616Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"b5cbe8d9-70ca-4eba-817f-8598d947e842 is now active with session: 4d5e62ed-927c-4701-bfb4-a02cc90610cf","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827256937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827269367Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310,"session_id":"4d5e62ed-927c-4701-bfb4-a02cc90610cf"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:50.827281947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 296 len 8192 data: 3 4 02/30 Write block 4 len 16384 data: 4 3 3 3 03/30 Write block 412 len 24576 data: 4 3 3 2 2 3 04/30 Read block 218 len 24576 05/30 Read block 420 len 8192 06/30 Write block 133 len 4096 data: 4 07/30 Read block 2 len 12288 08/30 Write block 252 len 20480 data: 4 3 3 5 5 09/30 Write block 22 len 20480 data: 6 6 7 5 4 10/30 Read block 418 len 16384 11/30 Read block 440 len 4096 12/30 Flush 13/30 Write block 468 len 8192 data: 3 4 14/30 Write block 331 len 20480 data: 4 4 4 4 4 15/30 Read block 103 len 36864 16/30 Read block 477 len 32768 17/30 Read block 550 len 20480 18/30 Flush 19/30 Write block 17 len 8192 data: 5 4 20/30 Write block 484 len 8192 data: 5 5 21/30 Read block 505 len 16384 22/30 Write block 258 len 20480 data: 5 4 3 2 2 23/30 Read block 575 len 8192 24/30 Read block 152 len 16384 25/30 Read block 236 len 24576 26/30 Write block 54 len 12288 data: 5 5 5 27/30 Read block 591 len 28672 28/30 Read block 318 len 40960 29/30 Read block 412 len 40960 30/30 Write block 315 len 12288 data: 5 5 5 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:50.844522259Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 54 54 54 T T T 1 020-039 32 32 32 54 54 54 F F F 2 040-059 31 31 31 53 53 53 T T T 3 060-079 31 31 31 53 53 53 F F F 4 080-099 25 25 25 46 46 46 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 30 30 30 52 52 52 F F F 12 240-259 32 32 32 54 54 54 T T T 13 260-279 30 30 30 52 52 52 T T T 14 280-299 32 32 32 54 54 54 F F F 15 300-319 26 26 26 48 48 48 T T T 16 320-339 32 32 32 55 55 55 F F F 17 340-359 31 31 31 53 53 53 F F F 18 360-379 29 29 29 51 51 51 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 32 32 32 54 54 54 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 32 32 32 55 55 55 F F F 24 480-499 26 26 26 48 48 48 T T T 25 500-519 26 26 26 47 47 47 F F F 26 520-539 29 29 29 51 51 51 F F F 27 540-559 30 30 30 52 52 52 F F F 28 560-579 31 31 31 53 53 53 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 32, Max flush: 55 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 33 33 33 56 56 56 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 25 25 25 46 46 46 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 29 29 29 51 51 51 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 30 30 30 52 52 52 F F F 12 240-259 33 33 33 56 56 56 F F F 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 54 54 54 F F F 15 300-319 33 33 33 56 56 56 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 31 31 31 53 53 53 F F F 18 360-379 29 29 29 51 51 51 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 32 32 32 54 54 54 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 32 32 32 55 55 55 F F F 24 480-499 33 33 33 56 56 56 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 29 29 29 51 51 51 F F F 27 540-559 30 30 30 52 52 52 F F F 28 560-579 31 31 31 53 53 53 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 33, Max flush: 56 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.137615302Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.179953424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.180552768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.180588559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.180603479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.18061899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318} {"msg":"Crucible fa3a4ab6-7b50-4425-ae23-4d527adda0c6 has session id: f42a0a4c-188c-491c-9f80-ae446ced4d8a","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.18104281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Upstairs opts: Upstairs UUID: fa3a4ab6-7b50-4425-ae23-4d527adda0c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.181088171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Crucible stats registered with UUID: fa3a4ab6-7b50-4425-ae23-4d527adda0c6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.181113982Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.210645707Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.210690799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318} {"msg":"fa3a4ab6-7b50-4425-ae23-4d527adda0c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.21075291Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.210785341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.210833422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.210910514Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.210959955Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211022356Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211064708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211250002Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211339144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211364255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48481)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211387005Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55461)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211420886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65158)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211449527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211708763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211736134Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211895777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211934688Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.211964009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.21199318Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.212462481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.212586084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.212726487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213076306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213107216Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213123927Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213185478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213205819Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213220009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213232699Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.21324631Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[0]R flush_numbers[0..12]: [56, 54, 56, 53, 46, 50, 54, 51, 50, 53, 35, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.21326041Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[0]R generation[0..12]: [33, 32, 33, 31, 25, 28, 32, 29, 28, 31, 20, 30]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.21327447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213288841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[1]R flush_numbers[0..12]: [56, 54, 56, 53, 46, 50, 54, 51, 50, 53, 35, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213306741Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[1]R generation[0..12]: [33, 32, 33, 31, 25, 28, 32, 29, 28, 31, 20, 30]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213321302Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213337762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[2]R flush_numbers[0..12]: [56, 54, 56, 53, 46, 50, 54, 51, 50, 53, 35, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213363583Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[2]R generation[0..12]: [33, 32, 33, 31, 25, 28, 32, 29, 28, 31, 20, 30]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213393063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:08:55.213419384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213454815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213482195Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213509066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213534747Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"":"downstairs","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213550687Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213566577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"0","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213592228Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"1","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213618569Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"client":"2","":"downstairs","session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213645659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.21366103Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"fa3a4ab6-7b50-4425-ae23-4d527adda0c6 is now active with session: f42a0a4c-188c-491c-9f80-ae446ced4d8a","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.21368548Z",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213712141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318,"session_id":"f42a0a4c-188c-491c-9f80-ae446ced4d8a"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:55.213734441Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 40960 data: 2 2 2 2 2 3 3 4 4 4 02/30 Read block 114 len 28672 03/30 Read block 93 len 4096 04/30 Read block 578 len 28672 05/30 Write block 129 len 16384 data: 3 3 4 4 06/30 Read block 68 len 36864 07/30 Write block 119 len 16384 data: 5 4 4 4 08/30 Read block 33 len 8192 09/30 Write block 50 len 12288 data: 5 4 6 10/30 Write block 345 len 12288 data: 3 3 3 11/30 Read block 264 len 36864 12/30 Write block 405 len 40960 data: 4 4 4 4 3 3 4 5 4 4 13/30 Write block 153 len 24576 data: 3 4 5 6 6 6 14/30 Write block 573 len 40960 data: 4 5 5 5 4 4 5 5 6 4 15/30 Read block 498 len 24576 16/30 Write block 232 len 12288 data: 4 4 4 17/30 Write block 249 len 4096 data: 3 18/30 Write block 84 len 12288 data: 2 2 2 19/30 Write block 487 len 32768 data: 5 4 4 4 4 2 2 2 20/30 Write block 100 len 20480 data: 7 7 8 7 6 21/30 Read block 86 len 4096 22/30 Write block 478 len 20480 data: 4 5 5 6 5 23/30 Read block 418 len 40960 24/30 Read block 133 len 12288 25/30 Read block 337 len 36864 26/30 Write block 536 len 36864 data: 5 6 5 4 5 5 5 6 5 27/30 Write block 477 len 20480 data: 3 5 6 6 7 28/30 Write block 28 len 40960 data: 3 2 3 3 2 2 2 2 3 5 29/30 Read block 375 len 20480 30/30 Read block 513 len 20480 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:55.23331754Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 32 32 32 54 54 54 T T T 2 040-059 33 33 33 56 56 56 T T T 3 060-079 31 31 31 53 53 53 F F F 4 080-099 25 25 25 46 46 46 T T T 5 100-119 28 28 28 50 50 50 T T T 6 120-139 32 32 32 54 54 54 T T T 7 140-159 29 29 29 51 51 51 T T T 8 160-179 28 28 28 50 50 50 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 30 30 30 52 52 52 T T T 12 240-259 33 33 33 56 56 56 T T T 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 54 54 54 F F F 15 300-319 33 33 33 56 56 56 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 31 31 31 53 53 53 T T T 18 360-379 29 29 29 51 51 51 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 32 32 32 54 54 54 T T T 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 32 32 32 55 55 55 T T T 24 480-499 33 33 33 56 56 56 T T T 25 500-519 26 26 26 47 47 47 F F F 26 520-539 29 29 29 51 51 51 T T T 27 540-559 30 30 30 52 52 52 T T T 28 560-579 31 31 31 53 53 53 T T T 29 580-599 28 28 28 50 50 50 T T T Max gen: 33, Max flush: 56 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 34 34 34 57 57 57 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 34 34 34 57 57 57 F F F 5 100-119 34 34 34 57 57 57 F F F 6 120-139 34 34 34 57 57 57 F F F 7 140-159 34 34 34 57 57 57 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 33 33 33 56 56 56 F F F 14 280-299 32 32 32 54 54 54 F F F 15 300-319 33 33 33 56 56 56 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 34 34 34 57 57 57 F F F 18 360-379 29 29 29 51 51 51 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 34 34 34 57 57 57 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 34 34 34 57 57 57 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 34 34 34 57 57 57 F F F 27 540-559 34 34 34 57 57 57 F F F 28 560-579 34 34 34 57 57 57 F F F 29 580-599 34 34 34 57 57 57 F F F Max gen: 34, Max flush: 57 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.547937776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:59.589170242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.58953402Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.620001049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.620561452Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.620596523Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.620610863Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.620625404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326} {"msg":"Crucible d1edb8f0-b189-463b-b6b6-9f070830c4e0 has session id: bfdd12b5-1585-4759-8cca-8c4ac4037325","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.621019903Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Upstairs opts: Upstairs UUID: d1edb8f0-b189-463b-b6b6-9f070830c4e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.621067774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Crucible stats registered with UUID: d1edb8f0-b189-463b-b6b6-9f070830c4e0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.621089355Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.650788255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.650818145Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326} {"msg":"d1edb8f0-b189-463b-b6b6-9f070830c4e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.650883317Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.650911138Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.650945919Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.650975059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.65100496Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651031681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651057911Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651087162Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651118073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651181964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_connection connected from Ok(127.0.0.1:59495)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651209405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_connection connected from Ok(127.0.0.1:56723)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651235006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_connection connected from Ok(127.0.0.1:35628)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651261756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651906322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651936562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651968813Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.651999294Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.652082856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.652120097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.652502556Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.652581098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.652900775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653123441Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653151431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653166472Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653179442Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653192912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653206253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653368426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653388477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 57, 53, 57, 57, 57, 57, 50, 53, 35, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653406337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[0]R generation[0..12]: [33, 34, 34, 31, 34, 34, 34, 34, 28, 31, 20, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653420818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653434958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 57, 53, 57, 57, 57, 57, 50, 53, 35, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653448888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[1]R generation[0..12]: [33, 34, 34, 31, 34, 34, 34, 34, 28, 31, 20, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653462439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653476559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 57, 53, 57, 57, 57, 57, 50, 53, 35, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653490209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[2]R generation[0..12]: [33, 34, 34, 31, 34, 34, 34, 34, 28, 31, 20, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.65350389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.65351786Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.6535311Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653544521Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653557671Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:08:59.653570661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"":"downstairs","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653589002Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653603082Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"0","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653616782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"1","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653630023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"client":"2","":"downstairs","session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653643583Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653656193Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"d1edb8f0-b189-463b-b6b6-9f070830c4e0 is now active with session: bfdd12b5-1585-4759-8cca-8c4ac4037325","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653669114Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653681194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326,"session_id":"bfdd12b5-1585-4759-8cca-8c4ac4037325"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:08:59.653693584Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 104 len 32768 data: 7 6 5 3 3 3 3 3 02/30 Write block 394 len 20480 data: 5 5 4 4 4 03/30 Read block 474 len 24576 04/30 Write block 520 len 8192 data: 3 3 05/30 Write block 494 len 40960 data: 3 2 2 2 2 2 3 2 2 2 06/30 Write block 483 len 12288 data: 5 6 6 07/30 Read block 483 len 8192 08/30 Flush 09/30 Write block 481 len 16384 data: 8 6 6 7 10/30 Read block 64 len 20480 11/30 Flush 12/30 Write block 378 len 40960 data: 6 5 4 4 4 3 3 3 2 2 13/30 Write block 153 len 28672 data: 4 5 6 7 7 7 7 14/30 Write block 361 len 4096 data: 3 15/30 Flush 16/30 Write block 285 len 8192 data: 4 5 17/30 Read block 94 len 40960 18/30 Read block 113 len 40960 19/30 Read block 5 len 8192 20/30 Write block 269 len 32768 data: 4 4 4 5 6 5 5 3 21/30 Write block 165 len 40960 data: 3 3 3 2 2 2 2 3 6 5 22/30 Write block 101 len 40960 data: 8 9 8 8 7 6 4 4 4 4 23/30 Read block 575 len 28672 24/30 Read block 311 len 8192 25/30 Write block 47 len 40960 data: 4 4 5 6 5 7 5 6 6 6 26/30 Read block 197 len 24576 27/30 Read block 136 len 32768 28/30 Write block 543 len 32768 data: 7 6 4 4 4 2 2 3 29/30 Write block 582 len 8192 data: 5 5 30/30 Read block 391 len 40960 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 2 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 15 15 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:08:59.673471087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 34 34 34 57 57 57 T T T 3 060-079 31 31 31 53 53 53 F F F 4 080-099 34 34 34 57 57 57 F F F 5 100-119 36 36 36 58 58 58 T T T 6 120-139 34 34 34 57 57 57 F F F 7 140-159 36 36 36 60 60 60 F F F 8 160-179 28 28 28 50 50 50 T T T 9 180-199 31 31 31 53 53 53 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 33 33 33 56 56 56 T T T 14 280-299 32 32 32 54 54 54 T T T 15 300-319 33 33 33 56 56 56 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 34 34 34 57 57 57 F F F 18 360-379 36 36 36 60 60 60 F F F 19 380-399 36 36 36 60 60 60 F F F 20 400-419 34 34 34 57 57 57 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 34 34 34 57 57 57 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 36 36 36 58 58 58 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 34 34 34 57 57 57 T T T 28 560-579 34 34 34 57 57 57 F F F 29 580-599 34 34 34 57 57 57 T T T Max gen: 36, Max flush: 60 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 37 37 37 61 61 61 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 34 34 34 57 57 57 F F F 5 100-119 37 37 37 61 61 61 F F F 6 120-139 34 34 34 57 57 57 F F F 7 140-159 36 36 36 60 60 60 F F F 8 160-179 37 37 37 61 61 61 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 20 20 20 35 35 35 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 37 37 37 61 61 61 F F F 15 300-319 33 33 33 56 56 56 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 34 34 34 57 57 57 F F F 18 360-379 36 36 36 60 60 60 F F F 19 380-399 36 36 36 60 60 60 F F F 20 400-419 34 34 34 57 57 57 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 27 27 27 49 49 49 F F F 23 460-479 34 34 34 57 57 57 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 36 36 36 58 58 58 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 37 37 37 61 61 61 F F F 28 560-579 34 34 34 57 57 57 F F F 29 580-599 37 37 37 61 61 61 F F F Max gen: 37, Max flush: 61 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:03.978136885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.021976144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.022868075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.022904776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.022919756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.022934616Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334} {"msg":"Crucible 2290a6d0-cd6a-4bcd-9a4b-969dc0a68262 has session id: 8b3803c2-7037-4899-9898-a47576b833c0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.023474269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Upstairs opts: Upstairs UUID: 2290a6d0-cd6a-4bcd-9a4b-969dc0a68262, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.02350985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Crucible stats registered with UUID: 2290a6d0-cd6a-4bcd-9a4b-969dc0a68262","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.023526521Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.05401919Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.05404741Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334} {"msg":"2290a6d0-cd6a-4bcd-9a4b-969dc0a68262 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054106832Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054125212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054153973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054169583Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054184783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054198734Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054212504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054229855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054258945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054286896Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64675)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054325787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58452)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054356687Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64119)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.054387678Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.055104045Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.055131766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.055245959Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.055266999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.05528681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.05530158Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.055808022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.055853033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.055989377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056374506Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056404736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056421287Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056434527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056448668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056462758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056490738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056506019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 61, 53, 57, 61, 57, 60, 61, 53, 35, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.0565507Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[0]R generation[0..12]: [33, 34, 37, 31, 34, 37, 34, 36, 37, 31, 20, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.05656853Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056586741Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 61, 53, 57, 61, 57, 60, 61, 53, 35, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056601351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[1]R generation[0..12]: [33, 34, 37, 31, 34, 37, 34, 36, 37, 31, 20, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056615261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056629682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 61, 53, 57, 61, 57, 60, 61, 53, 35, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056643402Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[2]R generation[0..12]: [33, 34, 37, 31, 34, 37, 34, 36, 37, 31, 20, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056657013Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056671293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056684593Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056697963Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Next flush: 62","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:09:04.056711474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056729864Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"":"downstairs","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056744125Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056756345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"0","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056769825Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"1","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056783025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"client":"2","":"downstairs","session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056798156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056810976Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"2290a6d0-cd6a-4bcd-9a4b-969dc0a68262 is now active with session: 8b3803c2-7037-4899-9898-a47576b833c0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056824106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056836427Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334,"session_id":"8b3803c2-7037-4899-9898-a47576b833c0"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:04.056848947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 344 len 16384 data: 5 4 4 4 02/30 Read block 263 len 12288 03/30 Flush 04/30 Write block 505 len 8192 data: 2 3 05/30 Flush 06/30 Write block 481 len 40960 data: 9 7 7 8 7 5 6 5 5 5 07/30 Write block 548 len 20480 data: 3 3 4 4 5 08/30 Write block 67 len 28672 data: 8 5 5 5 4 4 4 09/30 Read block 163 len 36864 10/30 Write block 443 len 8192 data: 2 3 11/30 Read block 12 len 20480 12/30 Read block 114 len 40960 13/30 Write block 366 len 12288 data: 3 3 4 14/30 Read block 549 len 24576 15/30 Write block 30 len 20480 data: 4 4 3 3 3 16/30 Read block 403 len 36864 17/30 Write block 310 len 36864 data: 3 3 2 4 5 6 6 6 4 18/30 Read block 42 len 40960 19/30 Flush 20/30 Write block 515 len 16384 data: 5 4 5 4 21/30 Write block 286 len 16384 data: 6 6 5 5 22/30 Write block 199 len 20480 data: 3 3 3 3 3 23/30 Flush 24/30 Write block 81 len 40960 data: 3 4 3 3 3 3 2 3 3 3 25/30 Write block 377 len 20480 data: 6 7 6 5 5 26/30 Flush 27/30 Write block 141 len 24576 data: 7 5 5 5 4 4 28/30 Read block 211 len 4096 29/30 Write block 32 len 32768 data: 4 4 4 3 4 6 6 6 30/30 Read block 479 len 16384 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:04.078649398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 38 38 38 64 64 64 T T T 2 040-059 37 37 37 61 61 61 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 38 38 38 66 66 66 F F F 5 100-119 37 37 37 61 61 61 F F F 6 120-139 34 34 34 57 57 57 F F F 7 140-159 36 36 36 60 60 60 T T T 8 160-179 37 37 37 61 61 61 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 38 38 38 65 65 65 F F F 15 300-319 38 38 38 64 64 64 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 66 66 66 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 34 34 34 57 57 57 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 34 34 34 57 57 57 F F F 24 480-499 38 38 38 64 64 64 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 38 38 38 64 64 64 F F F 28 560-579 34 34 34 57 57 57 F F F 29 580-599 37 37 37 61 61 61 F F F Max gen: 38, Max flush: 66 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 39 39 39 67 67 67 F F F 2 040-059 37 37 37 61 61 61 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 38 38 38 66 66 66 F F F 5 100-119 37 37 37 61 61 61 F F F 6 120-139 34 34 34 57 57 57 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 37 37 37 61 61 61 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 38 38 38 65 65 65 F F F 15 300-319 38 38 38 64 64 64 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 66 66 66 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 34 34 34 57 57 57 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 34 34 34 57 57 57 F F F 24 480-499 38 38 38 64 64 64 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 38 38 38 64 64 64 F F F 28 560-579 34 34 34 57 57 57 F F F 29 580-599 37 37 37 61 61 61 F F F Max gen: 39, Max flush: 67 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.321501619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.364345493Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.364861066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.364896517Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.364915907Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.364931367Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342} {"msg":"Crucible 820316a0-64a3-43d6-b1b7-cce936c00379 has session id: 72942133-9be1-4e5e-90a6-26cc8a690394","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.365358628Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Upstairs opts: Upstairs UUID: 820316a0-64a3-43d6-b1b7-cce936c00379, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.365401759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Crucible stats registered with UUID: 820316a0-64a3-43d6-b1b7-cce936c00379","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.365425649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.395873827Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.395909768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342} {"msg":"820316a0-64a3-43d6-b1b7-cce936c00379 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.3959902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396014781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396045541Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396064122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396081672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396098632Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396121093Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396158694Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396193995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396229336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_connection connected from Ok(127.0.0.1:55099)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396263876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_connection connected from Ok(127.0.0.1:41680)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.39639997Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_connection connected from Ok(127.0.0.1:57409)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.396442661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397022285Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397061275Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397087586Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397113077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397267491Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397297761Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397788843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397819094Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.397835434Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398373597Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398409388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398425558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398438479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398452099Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398465489Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.3984777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.39849124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[0]R flush_numbers[0..12]: [56, 67, 61, 64, 66, 61, 57, 67, 61, 65, 65, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.39850524Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[0]R generation[0..12]: [33, 39, 37, 38, 38, 37, 34, 39, 37, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398519041Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398533401Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[1]R flush_numbers[0..12]: [56, 67, 61, 64, 66, 61, 57, 67, 61, 65, 65, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398547241Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[1]R generation[0..12]: [33, 39, 37, 38, 38, 37, 34, 39, 37, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398561091Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398582662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[2]R flush_numbers[0..12]: [56, 67, 61, 64, 66, 61, 57, 67, 61, 65, 65, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398603912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[2]R generation[0..12]: [33, 39, 37, 38, 38, 37, 34, 39, 37, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398630883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398648004Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398661564Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398674924Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398695415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398718455Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"":"downstairs","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398734186Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398746536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"0","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398764836Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"1","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-12-09T15:09:08.398788937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"client":"2","":"downstairs","session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398808247Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398820938Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"820316a0-64a3-43d6-b1b7-cce936c00379 is now active with session: 72942133-9be1-4e5e-90a6-26cc8a690394","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398833838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398845858Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342,"session_id":"72942133-9be1-4e5e-90a6-26cc8a690394"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:08.398858218Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 8192 data: 2 3 02/30 Read block 539 len 4096 03/30 Read block 30 len 36864 04/30 Flush 05/30 Write block 157 len 36864 data: 8 8 8 6 6 5 2 3 4 06/30 Read block 591 len 24576 07/30 Write block 291 len 28672 data: 5 5 4 4 3 4 5 08/30 Read block 371 len 28672 09/30 Write block 168 len 36864 data: 3 3 3 3 4 7 6 5 5 10/30 Read block 517 len 20480 11/30 Write block 403 len 4096 data: 6 12/30 Write block 552 len 32768 data: 6 6 7 5 5 5 3 2 13/30 Read block 480 len 32768 14/30 Read block 272 len 16384 15/30 Write block 229 len 20480 data: 4 3 3 5 5 16/30 Write block 589 len 40960 data: 3 3 3 3 3 3 3 3 3 2 17/30 Read block 528 len 20480 18/30 Flush 19/30 Write block 127 len 8192 data: 4 3 20/30 Write block 297 len 28672 data: 6 3 3 3 4 4 4 21/30 Read block 328 len 20480 22/30 Read block 418 len 16384 23/30 Write block 159 len 20480 data: 9 7 7 6 3 24/30 Flush 25/30 Write block 224 len 4096 data: 6 26/30 Read block 162 len 28672 27/30 Write block 478 len 32768 data: 6 7 7 10 8 8 9 8 28/30 Read block 451 len 24576 29/30 Read block 226 len 4096 30/30 Read block 397 len 24576 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:08.41898676Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 39 39 39 67 67 67 F F F 2 040-059 37 37 37 61 61 61 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 38 38 38 66 66 66 F F F 5 100-119 40 40 40 68 68 68 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 40 40 40 70 70 70 F F F 8 160-179 40 40 40 70 70 70 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 40 40 40 69 69 69 T T T 12 240-259 34 34 34 57 57 57 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 40 40 40 70 70 70 F F F 15 300-319 40 40 40 70 70 70 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 66 66 66 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 40 40 40 69 69 69 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 34 34 34 57 57 57 T T T 24 480-499 38 38 38 64 64 64 T T T 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 34 34 34 57 57 57 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 40, Max flush: 70 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 F F F 1 020-039 39 39 39 67 67 67 F F F 2 040-059 37 37 37 61 61 61 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 38 38 38 66 66 66 F F F 5 100-119 40 40 40 68 68 68 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 40 40 40 70 70 70 F F F 8 160-179 40 40 40 70 70 70 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 41 41 41 71 71 71 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 40 40 40 70 70 70 F F F 15 300-319 40 40 40 70 70 70 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 66 66 66 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 40 40 40 69 69 69 F F F 21 420-439 29 29 29 51 51 51 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 41 41 41 71 71 71 F F F 24 480-499 41 41 41 71 71 71 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 34 34 34 57 57 57 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 41, Max flush: 71 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.667452395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.710527285Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.711125089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.71116006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.71117491Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.711189251Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350} {"msg":"Crucible f2d2ab34-e519-45e7-8e44-a35fce0a20c8 has session id: 7c784acb-fad8-4c9a-8cc7-52a43430e29b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.711658382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Upstairs opts: Upstairs UUID: f2d2ab34-e519-45e7-8e44-a35fce0a20c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.711707093Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Crucible stats registered with UUID: f2d2ab34-e519-45e7-8e44-a35fce0a20c8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.711732954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741526366Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741552766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350} {"msg":"f2d2ab34-e519-45e7-8e44-a35fce0a20c8 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741610838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741628908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741654429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741669549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.7416908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.74171234Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741737171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741763742Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741793022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.741822613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39019)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742020288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64902)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742060639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57493)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742079249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742478019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742518889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.74255077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742617192Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742646693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.742663503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743217386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743248247Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743278048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743817081Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743845611Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743860852Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743873652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743891372Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743905783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743918383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743931893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[0]R flush_numbers[0..12]: [56, 67, 61, 64, 66, 68, 70, 70, 70, 65, 65, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743946124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[0]R generation[0..12]: [33, 39, 37, 38, 38, 40, 40, 40, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743960094Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743974304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[1]R flush_numbers[0..12]: [56, 67, 61, 64, 66, 68, 70, 70, 70, 65, 65, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.743988155Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[1]R generation[0..12]: [33, 39, 37, 38, 38, 40, 40, 40, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744002725Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744016875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[2]R flush_numbers[0..12]: [56, 67, 61, 64, 66, 68, 70, 70, 70, 65, 65, 71]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744030706Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[2]R generation[0..12]: [33, 39, 37, 38, 38, 40, 40, 40, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744044386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744058436Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744071567Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO ":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744096657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744110468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744123828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"":"downstairs","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744137208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744149608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"0","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744162879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"1","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744176239Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"client":"2","":"downstairs","session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.74418964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.74420229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"f2d2ab34-e519-45e7-8e44-a35fce0a20c8 is now active with session: 7c784acb-fad8-4c9a-8cc7-52a43430e29b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.7442153Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.74422743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350,"session_id":"7c784acb-fad8-4c9a-8cc7-52a43430e29b"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:12.744243051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 407 len 20480 data: 5 5 4 4 5 02/30 Flush 03/30 Write block 193 len 28672 data: 2 3 3 3 4 4 4 04/30 Read block 327 len 24576 05/30 Read block 107 len 8192 06/30 Read block 416 len 36864 07/30 Read block 432 len 4096 08/30 Write block 427 len 28672 data: 3 3 3 3 3 3 3 09/30 Read block 523 len 28672 10/30 Write block 295 len 16384 data: 4 5 7 4 11/30 Read block 467 len 28672 12/30 Read block 206 len 12288 13/30 Read block 27 len 40960 14/30 Read block 228 len 16384 15/30 Flush 16/30 Write block 479 len 12288 data: 8 8 11 17/30 Write block 232 len 24576 data: 6 6 5 3 3 4 18/30 Write block 38 len 4096 data: 7 19/30 Flush 20/30 Write block 410 len 32768 data: 5 6 6 5 5 3 3 4 21/30 Flush 22/30 Write block 565 len 40960 data: 4 3 3 3 4 4 3 3 5 6 23/30 Read block 397 len 24576 24/30 Flush 25/30 Flush 26/30 Write block 369 len 16384 data: 4 4 4 4 27/30 Read block 119 len 40960 28/30 Write block 305 len 28672 data: 5 6 6 5 3 4 4 29/30 Write block 98 len 16384 data: 5 6 8 9 30/30 Write block 15 len 16384 data: 3 5 6 5 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 7 Done Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 3 8 Done 3 2 2 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:12.76386873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 56 56 56 T T T 1 020-039 42 42 42 74 74 74 F F F 2 040-059 37 37 37 61 61 61 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 38 38 38 66 66 66 T T T 5 100-119 40 40 40 68 68 68 T T T 6 120-139 40 40 40 70 70 70 F F F 7 140-159 40 40 40 70 70 70 F F F 8 160-179 40 40 40 70 70 70 F F F 9 180-199 42 42 42 73 73 73 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 42 42 42 74 74 74 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 42 42 42 73 73 73 F F F 15 300-319 40 40 40 70 70 70 T T T 16 320-339 32 32 32 55 55 55 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 66 66 66 T T T 19 380-399 38 38 38 66 66 66 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 42 42 42 74 74 74 F F F 24 480-499 42 42 42 74 74 74 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 42 42 42 76 76 76 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 42, Max flush: 76 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 77 77 77 F F F 1 020-039 42 42 42 74 74 74 F F F 2 040-059 37 37 37 61 61 61 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 43 43 43 77 77 77 F F F 5 100-119 43 43 43 77 77 77 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 40 40 40 70 70 70 F F F 8 160-179 40 40 40 70 70 70 F F F 9 180-199 42 42 42 73 73 73 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 42 42 42 74 74 74 F F F 12 240-259 34 34 34 57 57 57 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 42 42 42 73 73 73 F F F 15 300-319 43 43 43 77 77 77 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 43 43 43 77 77 77 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 42 42 42 74 74 74 F F F 24 480-499 42 42 42 74 74 74 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 42 42 42 76 76 76 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 43, Max flush: 77 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.024736131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.067230867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.067851982Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.067887553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.067901823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.067916283Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358} {"msg":"Crucible 57b5de77-b01c-4b4f-ae35-1d0ded80862a has session id: b421766e-5eaa-40b5-bca7-a5b7cdbf093e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.068342384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Upstairs opts: Upstairs UUID: 57b5de77-b01c-4b4f-ae35-1d0ded80862a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.068376605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Crucible stats registered with UUID: 57b5de77-b01c-4b4f-ae35-1d0ded80862a","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.068392445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098119325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098147006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358} {"msg":"57b5de77-b01c-4b4f-ae35-1d0ded80862a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098225188Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098249048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098274239Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098289169Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.09833237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098360101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098385042Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098411032Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098437723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51817)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098466234Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33053)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098494205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54994)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.098524475Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.099218602Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.099247612Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.099422377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.099463598Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.099501398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.099532119Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.09996969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100072062Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100143544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100539763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100564884Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100579524Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100655476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100689767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100706447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100719208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100732988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[0]R flush_numbers[0..12]: [77, 74, 61, 64, 77, 77, 70, 70, 70, 73, 65, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100746898Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[0]R generation[0..12]: [43, 42, 37, 38, 43, 43, 40, 40, 40, 42, 38, 42]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100761929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100779909Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[1]R flush_numbers[0..12]: [77, 74, 61, 64, 77, 77, 70, 70, 70, 73, 65, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100794329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[1]R generation[0..12]: [43, 42, 37, 38, 43, 43, 40, 40, 40, 42, 38, 42]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.10080826Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.10082263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[2]R flush_numbers[0..12]: [77, 74, 61, 64, 77, 77, 70, 70, 70, 73, 65, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100836401Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[2]R generation[0..12]: [43, 42, 37, 38, 43, 43, 40, 40, 40, 42, 38, 42]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100856221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-12-09T15:09:17.100934573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100957993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100972484Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100985764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.100998974Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"":"downstairs","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101012745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101024925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"0","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101038555Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"1","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101055856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"client":"2","":"downstairs","session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101069966Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101082566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"57b5de77-b01c-4b4f-ae35-1d0ded80862a is now active with session: b421766e-5eaa-40b5-bca7-a5b7cdbf093e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101095397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101107657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358,"session_id":"b421766e-5eaa-40b5-bca7-a5b7cdbf093e"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:17.101120127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 333 len 16384 data: 5 5 5 4 02/30 Write block 363 len 20480 data: 4 3 3 4 4 03/30 Write block 424 len 32768 data: 4 5 5 4 4 4 4 4 04/30 Write block 366 len 12288 data: 5 5 5 05/30 Read block 44 len 28672 06/30 Read block 344 len 16384 07/30 Read block 155 len 16384 08/30 Write block 90 len 12288 data: 4 3 3 09/30 Read block 203 len 8192 10/30 Write block 273 len 32768 data: 7 6 6 4 3 4 5 5 11/30 Read block 346 len 12288 12/30 Write block 160 len 4096 data: 8 13/30 Write block 352 len 24576 data: 3 3 3 3 4 3 14/30 Read block 144 len 16384 15/30 Write block 241 len 8192 data: 4 4 16/30 Write block 230 len 40960 data: 4 4 7 7 6 4 4 5 5 5 17/30 Write block 454 len 24576 data: 3 4 3 3 4 4 18/30 Write block 490 len 28672 data: 6 5 3 3 4 3 3 19/30 Read block 296 len 24576 20/30 Write block 567 len 16384 data: 4 4 5 5 21/30 Read block 50 len 8192 22/30 Flush 23/30 Write block 291 len 32768 data: 6 6 5 5 5 6 8 5 24/30 Write block 49 len 8192 data: 6 7 25/30 Flush 26/30 Write block 146 len 40960 data: 5 4 3 2 2 2 3 5 6 7 27/30 Read block 218 len 36864 28/30 Read block 52 len 4096 29/30 Read block 530 len 20480 30/30 Read block 236 len 40960 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:17.120118281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 77 77 77 F F F 1 020-039 42 42 42 74 74 74 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 44 44 44 78 78 78 F F F 5 100-119 43 43 43 77 77 77 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 40 40 40 70 70 70 T T T 8 160-179 44 44 44 78 78 78 F F F 9 180-199 42 42 42 73 73 73 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 44 44 44 78 78 78 F F F 12 240-259 44 44 44 78 78 78 F F F 13 260-279 44 44 44 78 78 78 F F F 14 280-299 44 44 44 79 79 79 F F F 15 300-319 43 43 43 77 77 77 F F F 16 320-339 44 44 44 78 78 78 F F F 17 340-359 44 44 44 78 78 78 F F F 18 360-379 44 44 44 78 78 78 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 44 44 44 78 78 78 F F F 23 460-479 42 42 42 74 74 74 F F F 24 480-499 44 44 44 78 78 78 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 44 44 44 78 78 78 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 44, Max flush: 79 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 77 77 77 F F F 1 020-039 42 42 42 74 74 74 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 38 38 38 64 64 64 F F F 4 080-099 44 44 44 78 78 78 F F F 5 100-119 43 43 43 77 77 77 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 45 45 45 80 80 80 F F F 8 160-179 44 44 44 78 78 78 F F F 9 180-199 42 42 42 73 73 73 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 44 44 44 78 78 78 F F F 12 240-259 44 44 44 78 78 78 F F F 13 260-279 44 44 44 78 78 78 F F F 14 280-299 44 44 44 79 79 79 F F F 15 300-319 43 43 43 77 77 77 F F F 16 320-339 44 44 44 78 78 78 F F F 17 340-359 44 44 44 78 78 78 F F F 18 360-379 44 44 44 78 78 78 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 44 44 44 78 78 78 F F F 23 460-479 42 42 42 74 74 74 F F F 24 480-499 44 44 44 78 78 78 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 44 44 44 78 78 78 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 45, Max flush: 80 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.357930762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.400709185Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.401381461Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.401420662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.401436692Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.401451432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366} {"msg":"Crucible 938287f7-79f9-4e2b-bce0-70c0de2beeda has session id: e421ca1a-fd79-4599-b411-fd98f6c37c16","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.401895013Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Upstairs opts: Upstairs UUID: 938287f7-79f9-4e2b-bce0-70c0de2beeda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.401942384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Crucible stats registered with UUID: 938287f7-79f9-4e2b-bce0-70c0de2beeda","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.401968025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431739196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431766227Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366} {"msg":"938287f7-79f9-4e2b-bce0-70c0de2beeda active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431842399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.43186127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.43188541Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.43190025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431914291Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431933021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431947741Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431961422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431975882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.431989782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_connection connected from Ok(127.0.0.1:59370)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.432072984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_connection connected from Ok(127.0.0.1:59050)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.432128646Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_connection connected from Ok(127.0.0.1:57388)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.432195478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433021687Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433049768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433066018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433080509Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433094279Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433108039Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433782935Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433823576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.433862447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434334609Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434363229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.43437938Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434451041Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434471422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434485702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434498253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434511963Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[0]R flush_numbers[0..12]: [77, 74, 79, 64, 78, 77, 70, 80, 78, 73, 65, 78]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434525893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[0]R generation[0..12]: [43, 42, 44, 38, 44, 43, 40, 45, 44, 42, 38, 44]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434540074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434554584Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[1]R flush_numbers[0..12]: [77, 74, 79, 64, 78, 77, 70, 80, 78, 73, 65, 78]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434568534Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[1]R generation[0..12]: [43, 42, 44, 38, 44, 43, 40, 45, 44, 42, 38, 44]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434582415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434600395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[2]R flush_numbers[0..12]: [77, 74, 79, 64, 78, 77, 70, 80, 78, 73, 65, 78]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434615035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[2]R generation[0..12]: [43, 42, 44, 38, 44, 43, 40, 45, 44, 42, 38, 44]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434629006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434643336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434656666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434670187Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434683327Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434705458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"":"downstairs","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434718918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434731098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"0","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434744658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"1","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434757949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"client":"2","":"downstairs","session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434771459Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.434783999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"938287f7-79f9-4e2b-bce0-70c0de2beeda is now active with session: e421ca1a-fd79-4599-b411-fd98f6c37c16","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.43479747Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.43480989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366,"session_id":"e421ca1a-fd79-4599-b411-fd98f6c37c16"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:21.43482414Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 283 len 28672 data: 4 4 5 7 7 6 6 02/30 Read block 20 len 16384 03/30 Read block 189 len 4096 04/30 Read block 315 len 4096 05/30 Read block 557 len 36864 06/30 Read block 484 len 16384 07/30 Read block 556 len 24576 08/30 Flush 09/30 Write block 8 len 36864 data: 3 3 3 2 2 2 3 4 6 10/30 Flush 11/30 Write block 193 len 12288 data: 3 4 4 12/30 Flush 13/30 Write block 153 len 12288 data: 6 7 8 14/30 Write block 25 len 20480 data: 6 5 5 4 3 15/30 Read block 393 len 16384 16/30 Read block 510 len 8192 17/30 Read block 485 len 12288 18/30 Read block 186 len 20480 19/30 Write block 77 len 32768 data: 5 5 4 3 4 5 4 4 20/30 Read block 572 len 40960 21/30 Write block 531 len 8192 data: 4 5 22/30 Read block 323 len 12288 23/30 Read block 294 len 40960 24/30 Read block 559 len 24576 25/30 Write block 215 len 28672 data: 2 2 2 2 3 4 4 26/30 Write block 443 len 24576 data: 3 4 3 3 3 3 27/30 Read block 82 len 16384 28/30 Read block 183 len 32768 29/30 Write block 571 len 8192 data: 4 4 30/30 Read block 47 len 20480 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 17 18 18 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:21.454780117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 82 82 82 F F F 1 020-039 42 42 42 74 74 74 T T T 2 040-059 44 44 44 79 79 79 F F F 3 060-079 38 38 38 64 64 64 T T T 4 080-099 44 44 44 78 78 78 T T T 5 100-119 43 43 43 77 77 77 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 45 45 45 80 80 80 T T T 8 160-179 44 44 44 78 78 78 F F F 9 180-199 46 46 46 83 83 83 F F F 10 200-219 38 38 38 65 65 65 T T T 11 220-239 44 44 44 78 78 78 T T T 12 240-259 44 44 44 78 78 78 F F F 13 260-279 44 44 44 78 78 78 F F F 14 280-299 46 46 46 81 81 81 F F F 15 300-319 43 43 43 77 77 77 F F F 16 320-339 44 44 44 78 78 78 F F F 17 340-359 44 44 44 78 78 78 F F F 18 360-379 44 44 44 78 78 78 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 44 44 44 78 78 78 T T T 23 460-479 42 42 42 74 74 74 F F F 24 480-499 44 44 44 78 78 78 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 36 36 36 58 58 58 T T T 27 540-559 40 40 40 69 69 69 F F F 28 560-579 44 44 44 78 78 78 T T T 29 580-599 40 40 40 69 69 69 F F F Max gen: 46, Max flush: 83 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 82 82 82 F F F 1 020-039 47 47 47 84 84 84 F F F 2 040-059 44 44 44 79 79 79 F F F 3 060-079 47 47 47 84 84 84 F F F 4 080-099 47 47 47 84 84 84 F F F 5 100-119 43 43 43 77 77 77 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 47 47 47 84 84 84 F F F 8 160-179 44 44 44 78 78 78 F F F 9 180-199 46 46 46 83 83 83 F F F 10 200-219 47 47 47 84 84 84 F F F 11 220-239 47 47 47 84 84 84 F F F 12 240-259 44 44 44 78 78 78 F F F 13 260-279 44 44 44 78 78 78 F F F 14 280-299 46 46 46 81 81 81 F F F 15 300-319 43 43 43 77 77 77 F F F 16 320-339 44 44 44 78 78 78 F F F 17 340-359 44 44 44 78 78 78 F F F 18 360-379 44 44 44 78 78 78 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 42 42 42 74 74 74 F F F 24 480-499 44 44 44 78 78 78 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 47, Max flush: 84 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.736854646Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.779185857Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.779762321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.779799292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.779817983Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.779834213Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374} {"msg":"Crucible 05b8ae9c-16d8-478e-8d2e-f68c9084c567 has session id: 4b7d8905-bce6-4b77-85eb-7f749f808010","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.780283774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Upstairs opts: Upstairs UUID: 05b8ae9c-16d8-478e-8d2e-f68c9084c567, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.780331635Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Crucible stats registered with UUID: 05b8ae9c-16d8-478e-8d2e-f68c9084c567","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.780350615Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810182758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810215619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374} {"msg":"05b8ae9c-16d8-478e-8d2e-f68c9084c567 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.81024975Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.81026656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810296251Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810314361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810331722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810349702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810381533Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810413374Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810436904Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810455845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_connection connected from Ok(127.0.0.1:63868)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810476316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_connection connected from Ok(127.0.0.1:42916)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810510526Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_connection connected from Ok(127.0.0.1:42976)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.810545327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.811334616Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.811379667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.811415308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.811446559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.811477019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.81150415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.81191802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.81194628Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812204527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812463973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812492934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812508204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812533845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812548385Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812561945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812699869Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812719769Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[0]R flush_numbers[0..12]: [82, 84, 79, 84, 84, 77, 70, 84, 78, 83, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812734079Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[0]R generation[0..12]: [46, 47, 44, 47, 47, 43, 40, 47, 44, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.81274795Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.81276239Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[1]R flush_numbers[0..12]: [82, 84, 79, 84, 84, 77, 70, 84, 78, 83, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812776031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[1]R generation[0..12]: [46, 47, 44, 47, 47, 43, 40, 47, 44, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812789701Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812803911Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[2]R flush_numbers[0..12]: [82, 84, 79, 84, 84, 77, 70, 84, 78, 83, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812817621Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[2]R generation[0..12]: [46, 47, 44, 47, 47, 43, 40, 47, 44, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812831262Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812849062Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812862993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812876283Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812889403Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:09:25.812902403Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"":"downstairs","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812927004Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812951865Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"0","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.812976405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"1","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.813001406Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"client":"2","":"downstairs","session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.813027066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :30,"time":"2024-12-09T15:09:25.813053467Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"05b8ae9c-16d8-478e-8d2e-f68c9084c567 is now active with session: 4b7d8905-bce6-4b77-85eb-7f749f808010","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.813069018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.813081338Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374,"session_id":"4b7d8905-bce6-4b77-85eb-7f749f808010"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:25.813093888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 470 len 8192 data: 4 4 03/30 Read block 96 len 40960 04/30 Write block 162 len 20480 data: 7 4 4 5 4 05/30 Flush 06/30 Write block 345 len 24576 data: 5 5 5 3 3 3 07/30 Read block 438 len 24576 08/30 Write block 384 len 24576 data: 4 4 3 3 2 3 09/30 Write block 25 len 24576 data: 7 6 6 5 4 5 10/30 Write block 322 len 24576 data: 3 3 3 3 4 4 11/30 Read block 306 len 20480 12/30 Write block 368 len 8192 data: 6 5 13/30 Write block 402 len 32768 data: 5 7 7 5 5 6 6 5 14/30 Read block 99 len 16384 15/30 Flush 16/30 Write block 258 len 40960 data: 6 5 4 3 3 2 2 2 4 5 17/30 Read block 401 len 28672 18/30 Flush 19/30 Write block 154 len 24576 data: 8 9 8 9 9 10 20/30 Read block 546 len 28672 21/30 Write block 494 len 4096 data: 5 22/30 Read block 98 len 16384 23/30 Read block 194 len 4096 24/30 Read block 62 len 12288 25/30 Read block 566 len 12288 26/30 Write block 41 len 16384 data: 6 6 6 5 27/30 Write block 407 len 36864 data: 7 7 6 6 7 7 6 6 4 28/30 Write block 98 len 16384 data: 6 7 9 10 29/30 Write block 11 len 32768 data: 3 3 3 4 5 7 7 6 30/30 Write block 95 len 24576 data: 5 5 5 7 8 10 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:25.831322074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 82 82 82 T T T 1 020-039 48 48 48 87 87 87 F F F 2 040-059 44 44 44 79 79 79 T T T 3 060-079 47 47 47 84 84 84 F F F 4 080-099 47 47 47 84 84 84 T T T 5 100-119 43 43 43 77 77 77 T T T 6 120-139 40 40 40 70 70 70 F F F 7 140-159 47 47 47 84 84 84 T T T 8 160-179 48 48 48 86 86 86 F F F 9 180-199 46 46 46 83 83 83 F F F 10 200-219 47 47 47 84 84 84 F F F 11 220-239 47 47 47 84 84 84 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 46 46 46 81 81 81 F F F 15 300-319 43 43 43 77 77 77 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 48 48 48 87 87 87 F F F 18 360-379 48 48 48 87 87 87 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 48 48 48 87 87 87 T T T 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 44 44 44 78 78 78 T T T 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 48, Max flush: 88 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 89 89 89 F F F 1 020-039 48 48 48 87 87 87 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 47 47 47 84 84 84 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 49 49 49 89 89 89 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 49 49 49 89 89 89 F F F 8 160-179 48 48 48 86 86 86 F F F 9 180-199 46 46 46 83 83 83 F F F 10 200-219 47 47 47 84 84 84 F F F 11 220-239 47 47 47 84 84 84 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 46 46 46 81 81 81 F F F 15 300-319 43 43 43 77 77 77 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 48 48 48 87 87 87 F F F 18 360-379 48 48 48 87 87 87 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 49 49 49 89 89 89 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 49 49 49 89 89 89 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 40 40 40 69 69 69 F F F Max gen: 49, Max flush: 89 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.099244834Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.142787175Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.1433919Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.14342966Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.143445481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.143461141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382} {"msg":"Crucible 5622d5f4-3c71-4ef8-95ef-32aa64218a83 has session id: 6941b106-3d47-4818-bedc-f7ec71196916","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.143880721Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Upstairs opts: Upstairs UUID: 5622d5f4-3c71-4ef8-95ef-32aa64218a83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.143930592Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Crucible stats registered with UUID: 5622d5f4-3c71-4ef8-95ef-32aa64218a83","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.143957543Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173652103Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173687724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382} {"msg":"5622d5f4-3c71-4ef8-95ef-32aa64218a83 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173719255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173737665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173768566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173787396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173804727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173832867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173866198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173892549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173913389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.17394504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_connection connected from Ok(127.0.0.1:44547)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.173974791Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_connection connected from Ok(127.0.0.1:44535)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.174012972Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_connection connected from Ok(127.0.0.1:59609)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.174047943Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.17478028Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.174819011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.174849302Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.174876942Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.174904263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.174934474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.175383644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.17559432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.175746823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.1760507Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176073551Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176087651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176153713Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176176793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176192094Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176211494Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176225224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[0]R flush_numbers[0..12]: [89, 87, 89, 84, 89, 89, 70, 89, 86, 83, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176264065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[0]R generation[0..12]: [49, 48, 49, 47, 49, 49, 40, 49, 48, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176281666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176296136Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[1]R flush_numbers[0..12]: [89, 87, 89, 84, 89, 89, 70, 89, 86, 83, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176310006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[1]R generation[0..12]: [49, 48, 49, 47, 49, 49, 40, 49, 48, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176323647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176337777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[2]R flush_numbers[0..12]: [89, 87, 89, 84, 89, 89, 70, 89, 86, 83, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176351458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[2]R generation[0..12]: [49, 48, 49, 47, 49, 49, 40, 49, 48, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176365138Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176379208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176392519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176409209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176422949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:09:30.17643637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"":"downstairs","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.17646085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176486151Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"0","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176512722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"1","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176535772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"client":"2","":"downstairs","session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176550502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176563163Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 5622d5f4-3c71-4ef8-95ef-32aa64218a83 is now active with session: 6941b106-3d47-4818-bedc-f7ec71196916","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176578793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176591013Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382,"session_id":"6941b106-3d47-4818-bedc-f7ec71196916"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:30.176603364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 8192 data: 3 3 02/30 Write block 308 len 24576 data: 6 4 5 5 3 5 03/30 Read block 362 len 8192 04/30 Write block 356 len 40960 data: 5 4 3 3 3 4 3 5 4 4 05/30 Read block 224 len 28672 06/30 Write block 587 len 28672 data: 2 2 4 4 4 4 4 07/30 Read block 548 len 16384 08/30 Read block 42 len 12288 09/30 Read block 400 len 36864 10/30 Read block 458 len 28672 11/30 Read block 423 len 40960 12/30 Write block 221 len 20480 data: 5 5 5 7 7 13/30 Read block 514 len 12288 14/30 Read block 218 len 32768 15/30 Write block 197 len 24576 data: 5 5 5 4 4 4 16/30 Read block 395 len 16384 17/30 Read block 30 len 40960 18/30 Write block 192 len 16384 data: 2 4 5 5 19/30 Read block 135 len 40960 20/30 Write block 39 len 20480 data: 7 6 7 7 7 21/30 Read block 488 len 8192 22/30 Write block 156 len 36864 data: 9 10 10 11 9 8 8 5 5 23/30 Read block 388 len 20480 24/30 Write block 64 len 32768 data: 6 6 7 9 6 6 6 5 25/30 Read block 577 len 8192 26/30 Write block 153 len 32768 data: 7 9 10 10 11 11 12 10 27/30 Read block 200 len 16384 28/30 Write block 55 len 24576 data: 7 7 5 4 4 4 29/30 Read block 217 len 16384 30/30 Read block 355 len 20480 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 29 30 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:30.198505047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 89 89 89 F F F 1 020-039 48 48 48 87 87 87 T T T 2 040-059 49 49 49 89 89 89 T T T 3 060-079 47 47 47 84 84 84 T T T 4 080-099 49 49 49 89 89 89 F F F 5 100-119 49 49 49 89 89 89 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 49 49 49 89 89 89 T T T 8 160-179 48 48 48 86 86 86 T T T 9 180-199 46 46 46 83 83 83 T T T 10 200-219 47 47 47 84 84 84 T T T 11 220-239 47 47 47 84 84 84 T T T 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 46 46 46 81 81 81 F F F 15 300-319 43 43 43 77 77 77 T T T 16 320-339 48 48 48 87 87 87 F F F 17 340-359 48 48 48 87 87 87 T T T 18 360-379 48 48 48 87 87 87 T T T 19 380-399 48 48 48 87 87 87 F F F 20 400-419 49 49 49 89 89 89 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 49 49 49 89 89 89 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 40 40 40 69 69 69 T T T Max gen: 49, Max flush: 89 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 89 89 89 F F F 1 020-039 50 50 50 90 90 90 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 50 50 50 90 90 90 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 49 49 49 89 89 89 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 50 50 50 90 90 90 F F F 8 160-179 50 50 50 90 90 90 F F F 9 180-199 50 50 50 90 90 90 F F F 10 200-219 50 50 50 90 90 90 F F F 11 220-239 50 50 50 90 90 90 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 46 46 46 81 81 81 F F F 15 300-319 50 50 50 90 90 90 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 50 50 50 90 90 90 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 49 49 49 89 89 89 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 49 49 49 89 89 89 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 50 50 50 90 90 90 F F F Max gen: 50, Max flush: 90 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.492239835Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.534408753Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.535028698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.535064878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.535079639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.535094279Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390} {"msg":"Crucible 50140e94-66f7-4d46-b0e1-7e46ca447fc8 has session id: 7149c636-d3a2-40d6-a039-6371e13b476e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.53553224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Upstairs opts: Upstairs UUID: 50140e94-66f7-4d46-b0e1-7e46ca447fc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.535570601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Crucible stats registered with UUID: 50140e94-66f7-4d46-b0e1-7e46ca447fc8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.535588661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.565902726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.565936856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390} {"msg":"50140e94-66f7-4d46-b0e1-7e46ca447fc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.565988348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566008918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566048399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.56607001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.56608836Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566106951Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566125101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566142801Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566160682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566179242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55655)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566197393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65340)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566238144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53045)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.566279295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567165466Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567195426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567212207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567226667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567302219Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.56733419Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567823052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567960755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.567988116Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568328704Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568369595Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568399975Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568562739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.56859175Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.56860783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568621481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568636361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[0]R flush_numbers[0..12]: [89, 90, 90, 90, 89, 89, 70, 90, 90, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568651741Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[0]R generation[0..12]: [49, 50, 50, 50, 49, 49, 40, 50, 50, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568665842Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568681512Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[1]R flush_numbers[0..12]: [89, 90, 90, 90, 89, 89, 70, 90, 90, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568695852Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[1]R generation[0..12]: [49, 50, 50, 50, 49, 49, 40, 50, 50, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568709983Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568724603Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[2]R flush_numbers[0..12]: [89, 90, 90, 90, 89, 89, 70, 90, 90, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568738863Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[2]R generation[0..12]: [49, 50, 50, 50, 49, 49, 40, 50, 50, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568752614Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568766914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Max found gen is 51","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568789495Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Generation requested: 52 >= found:51","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568804965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568819725Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568834086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"":"downstairs","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568848656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568861436Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"0","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568875777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"1","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568889647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"client":"2","":"downstairs","session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568903717Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568920778Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"50140e94-66f7-4d46-b0e1-7e46ca447fc8 is now active with session: 7149c636-d3a2-40d6-a039-6371e13b476e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568934348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568947608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390,"session_id":"7149c636-d3a2-40d6-a039-6371e13b476e"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:34.568961619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 32768 data: 3 4 3 3 3 3 4 6 02/30 Write block 386 len 8192 data: 4 4 03/30 Read block 527 len 36864 04/30 Write block 223 len 36864 data: 6 8 8 6 6 4 5 5 5 05/30 Read block 167 len 32768 06/30 Read block 12 len 20480 07/30 Write block 158 len 36864 data: 12 13 11 9 9 6 6 6 5 08/30 Write block 9 len 8192 data: 4 4 09/30 Write block 205 len 16384 data: 2 2 3 4 10/30 Write block 343 len 28672 data: 5 6 6 6 6 4 4 11/30 Flush 12/30 Write block 65 len 8192 data: 7 8 13/30 Read block 61 len 28672 14/30 Read block 388 len 32768 15/30 Read block 239 len 12288 16/30 Read block 310 len 36864 17/30 Read block 422 len 20480 18/30 Read block 88 len 24576 19/30 Write block 294 len 20480 data: 6 6 7 9 6 20/30 Read block 480 len 4096 21/30 Write block 486 len 8192 data: 6 7 22/30 Read block 464 len 24576 23/30 Read block 458 len 4096 24/30 Read block 287 len 16384 25/30 Read block 508 len 40960 26/30 Write block 395 len 20480 data: 6 5 5 5 3 27/30 Read block 54 len 8192 28/30 Read block 168 len 8192 29/30 Write block 589 len 4096 data: 5 30/30 Write block 583 len 12288 data: 6 4 3 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 2 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:34.587240106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 91 91 91 F F F 1 020-039 50 50 50 90 90 90 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 50 50 50 90 90 90 T T T 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 52 52 52 91 91 91 F F F 8 160-179 52 52 52 91 91 91 F F F 9 180-199 50 50 50 90 90 90 F F F 10 200-219 52 52 52 91 91 91 F F F 11 220-239 52 52 52 91 91 91 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 46 46 46 81 81 81 T T T 15 300-319 50 50 50 90 90 90 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 52 52 52 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 52 52 52 91 91 91 T T T 20 400-419 49 49 49 89 89 89 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 49 49 49 89 89 89 T T T 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 50 50 50 90 90 90 T T T Max gen: 52, Max flush: 91 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 91 91 91 F F F 1 020-039 50 50 50 90 90 90 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 53 53 53 92 92 92 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 40 40 40 70 70 70 F F F 7 140-159 52 52 52 91 91 91 F F F 8 160-179 52 52 52 91 91 91 F F F 9 180-199 50 50 50 90 90 90 F F F 10 200-219 52 52 52 91 91 91 F F F 11 220-239 52 52 52 91 91 91 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 53 53 53 92 92 92 F F F 15 300-319 50 50 50 90 90 90 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 52 52 52 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 53 53 53 92 92 92 F F F 20 400-419 49 49 49 89 89 89 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 53 53 53 92 92 92 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 53 53 53 92 92 92 F F F Max gen: 53, Max flush: 92 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.843989949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.887637082Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.888299828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.888341119Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.88836551Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.88838535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398} {"msg":"Crucible 78ceafca-c959-48f5-abb6-badfb6775a35 has session id: b5b61a03-f972-4220-8ea9-347b750daaa5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.88880372Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Upstairs opts: Upstairs UUID: 78ceafca-c959-48f5-abb6-badfb6775a35, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.888848261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Crucible stats registered with UUID: 78ceafca-c959-48f5-abb6-badfb6775a35","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.888877432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918532311Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918556471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398} {"msg":"78ceafca-c959-48f5-abb6-badfb6775a35 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918641403Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918659794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918685074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918699745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918713515Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918727055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918740536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918754026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918768176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918782007Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54219)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918860699Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54446)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.918898529Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53623)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.919065293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.919705879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.919733799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.91975275Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.9197667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.91978007Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.919799561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.920407036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.920489727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.920520788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.920967059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.92101174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.92102961Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921071791Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921089912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921103492Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921119402Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921133683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[0]R flush_numbers[0..12]: [91, 90, 90, 92, 89, 91, 70, 91, 91, 90, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921147443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[0]R generation[0..12]: [52, 50, 50, 53, 49, 52, 40, 52, 52, 50, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921161674Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921175914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[1]R flush_numbers[0..12]: [91, 90, 90, 92, 89, 91, 70, 91, 91, 90, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921189664Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[1]R generation[0..12]: [52, 50, 50, 53, 49, 52, 40, 52, 52, 50, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921203184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921217155Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[2]R flush_numbers[0..12]: [91, 90, 90, 92, 89, 91, 70, 91, 91, 90, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921230815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[2]R generation[0..12]: [52, 50, 50, 53, 49, 52, 40, 52, 52, 50, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921244496Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921258346Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921271466Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921284626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921297637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:09:38.921312477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"":"downstairs","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921333968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921346198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"0","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921359428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"1","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921372648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"client":"2","":"downstairs","session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921385969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921398379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"78ceafca-c959-48f5-abb6-badfb6775a35 is now active with session: b5b61a03-f972-4220-8ea9-347b750daaa5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.921411229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.92142342Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398,"session_id":"b5b61a03-f972-4220-8ea9-347b750daaa5"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:38.92143566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 204 len 40960 data: 2 3 3 4 5 4 2 2 2 2 02/30 Read block 528 len 16384 03/30 Read block 291 len 32768 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 282 len 24576 data: 4 5 5 6 8 8 08/30 Read block 508 len 20480 09/30 Write block 158 len 40960 data: 13 14 12 10 10 7 7 7 6 4 10/30 Read block 506 len 4096 11/30 Write block 522 len 8192 data: 3 3 12/30 Read block 2 len 4096 13/30 Read block 397 len 16384 14/30 Read block 478 len 12288 15/30 Write block 25 len 24576 data: 8 7 7 6 5 6 16/30 Read block 123 len 36864 17/30 Read block 172 len 12288 18/30 Read block 216 len 12288 19/30 Read block 453 len 36864 20/30 Write block 318 len 8192 data: 5 5 21/30 Read block 277 len 36864 22/30 Read block 461 len 20480 23/30 Read block 362 len 40960 24/30 Read block 161 len 12288 25/30 Write block 599 len 4096 data: 2 26/30 Read block 442 len 24576 27/30 Read block 178 len 28672 28/30 Read block 496 len 40960 29/30 Write block 43 len 20480 data: 8 6 6 7 5 30/30 Write block 123 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:38.941223363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 91 91 91 F F F 1 020-039 50 50 50 90 90 90 T T T 2 040-059 50 50 50 90 90 90 T T T 3 060-079 53 53 53 92 92 92 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 40 40 40 70 70 70 T T T 7 140-159 52 52 52 91 91 91 T T T 8 160-179 52 52 52 91 91 91 T T T 9 180-199 50 50 50 90 90 90 F F F 10 200-219 54 54 54 93 93 93 F F F 11 220-239 52 52 52 91 91 91 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 53 53 53 92 92 92 T T T 15 300-319 50 50 50 90 90 90 T T T 16 320-339 48 48 48 87 87 87 F F F 17 340-359 52 52 52 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 53 53 53 92 92 92 F F F 20 400-419 49 49 49 89 89 89 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 53 53 53 92 92 92 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 47 47 47 84 84 84 T T T 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 53 53 53 92 92 92 T T T Max gen: 54, Max flush: 93 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 91 91 91 F F F 1 020-039 55 55 55 94 94 94 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 53 53 53 92 92 92 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 55 55 55 94 94 94 F F F 8 160-179 55 55 55 94 94 94 F F F 9 180-199 50 50 50 90 90 90 F F F 10 200-219 54 54 54 93 93 93 F F F 11 220-239 52 52 52 91 91 91 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 55 55 55 94 94 94 F F F 15 300-319 55 55 55 94 94 94 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 52 52 52 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 53 53 53 92 92 92 F F F 20 400-419 49 49 49 89 89 89 F F F 21 420-439 44 44 44 78 78 78 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 53 53 53 92 92 92 F F F 25 500-519 38 38 38 65 65 65 F F F 26 520-539 55 55 55 94 94 94 F F F 27 540-559 40 40 40 69 69 69 F F F 28 560-579 47 47 47 84 84 84 F F F 29 580-599 55 55 55 94 94 94 F F F Max gen: 55, Max flush: 94 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.22239427Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.266149866Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.266754191Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.266786631Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.266801242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.266815622Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406} {"msg":"Crucible 28ca2742-3363-4040-90ac-432b0fa24b1d has session id: 931e8ace-691e-447a-b37d-52689aa3be11","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.267243652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Upstairs opts: Upstairs UUID: 28ca2742-3363-4040-90ac-432b0fa24b1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.267291913Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Crucible stats registered with UUID: 28ca2742-3363-4040-90ac-432b0fa24b1d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.267318034Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.296950862Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.296977893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406} {"msg":"28ca2742-3363-4040-90ac-432b0fa24b1d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297039084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297067365Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297092416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297107256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297121126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297135307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297151057Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297177938Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297205219Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297232109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_connection connected from Ok(127.0.0.1:63942)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297447874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_connection connected from Ok(127.0.0.1:58401)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297491825Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_connection connected from Ok(127.0.0.1:59074)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.297519066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.298056729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.29808816Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.29810388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.298120901Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.298233523Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.298277364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.298767546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.298812547Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.298842638Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299318679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.29934737Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.29936287Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299431672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299452552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299466723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299479233Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299496683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[0]R flush_numbers[0..12]: [91, 94, 94, 92, 89, 91, 94, 94, 94, 90, 93, 91]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299511324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[0]R generation[0..12]: [52, 55, 55, 53, 49, 52, 55, 55, 55, 50, 54, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299525134Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299539164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[1]R flush_numbers[0..12]: [91, 94, 94, 92, 89, 91, 94, 94, 94, 90, 93, 91]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299552925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[1]R generation[0..12]: [52, 55, 55, 53, 49, 52, 55, 55, 55, 50, 54, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299566615Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299580645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[2]R flush_numbers[0..12]: [91, 94, 94, 92, 89, 91, 94, 94, 94, 90, 93, 91]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299594456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[2]R generation[0..12]: [52, 55, 55, 53, 49, 52, 55, 55, 55, 50, 54, 52]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299607896Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299621706Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299634767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:09:43.299648007Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299667277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299680408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"":"downstairs","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299695678Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299708298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"0","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299721649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"1","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299734939Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"client":"2","":"downstairs","session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299748209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.29976037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"28ca2742-3363-4040-90ac-432b0fa24b1d is now active with session: 931e8ace-691e-447a-b37d-52689aa3be11","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.29977317Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.29978523Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406,"session_id":"931e8ace-691e-447a-b37d-52689aa3be11"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:43.299797501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 211 len 24576 data: 3 3 3 2 3 3 02/30 Read block 140 len 16384 03/30 Read block 549 len 8192 04/30 Write block 501 len 32768 data: 3 3 3 2 3 4 4 4 05/30 Write block 397 len 32768 data: 6 6 4 3 4 6 8 8 06/30 Read block 512 len 24576 07/30 Read block 354 len 36864 08/30 Flush 09/30 Write block 507 len 40960 data: 5 5 4 5 4 4 4 5 6 5 10/30 Read block 233 len 24576 11/30 Write block 429 len 16384 data: 5 5 5 4 12/30 Flush 13/30 Write block 502 len 40960 data: 4 4 3 4 5 6 6 5 6 5 14/30 Read block 305 len 8192 15/30 Read block 448 len 40960 16/30 Read block 331 len 16384 17/30 Read block 96 len 8192 18/30 Read block 99 len 16384 19/30 Write block 231 len 12288 data: 6 8 8 20/30 Read block 465 len 16384 21/30 Write block 554 len 20480 data: 8 6 6 6 4 22/30 Write block 359 len 32768 data: 4 4 5 4 6 5 5 6 23/30 Write block 578 len 36864 data: 5 6 6 7 6 7 5 4 3 24/30 Read block 239 len 24576 25/30 Read block 379 len 4096 26/30 Write block 176 len 12288 data: 6 5 5 27/30 Read block 417 len 32768 28/30 Write block 148 len 12288 data: 4 3 3 29/30 Read block 284 len 20480 30/30 Write block 346 len 36864 data: 7 7 5 5 4 3 4 4 4 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 10 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:43.320333872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 91 91 91 F F F 1 020-039 55 55 55 94 94 94 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 53 53 53 92 92 92 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 55 55 55 94 94 94 T T T 8 160-179 55 55 55 94 94 94 T T T 9 180-199 50 50 50 90 90 90 F F F 10 200-219 56 56 56 95 95 95 F F F 11 220-239 52 52 52 91 91 91 T T T 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 55 55 55 94 94 94 F F F 15 300-319 55 55 55 94 94 94 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 52 52 52 91 91 91 T T T 18 360-379 50 50 50 90 90 90 T T T 19 380-399 56 56 56 95 95 95 F F F 20 400-419 56 56 56 95 95 95 F F F 21 420-439 56 56 56 96 96 96 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 53 53 53 92 92 92 F F F 25 500-519 56 56 56 96 96 96 T T T 26 520-539 55 55 55 94 94 94 F F F 27 540-559 40 40 40 69 69 69 T T T 28 560-579 47 47 47 84 84 84 T T T 29 580-599 55 55 55 94 94 94 T T T Max gen: 56, Max flush: 96 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 91 91 91 F F F 1 020-039 55 55 55 94 94 94 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 53 53 53 92 92 92 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 57 57 57 97 97 97 F F F 8 160-179 57 57 57 97 97 97 F F F 9 180-199 50 50 50 90 90 90 F F F 10 200-219 56 56 56 95 95 95 F F F 11 220-239 57 57 57 97 97 97 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 55 55 55 94 94 94 F F F 15 300-319 55 55 55 94 94 94 F F F 16 320-339 48 48 48 87 87 87 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 57 57 57 97 97 97 F F F 19 380-399 56 56 56 95 95 95 F F F 20 400-419 56 56 56 95 95 95 F F F 21 420-439 56 56 56 96 96 96 F F F 22 440-459 47 47 47 84 84 84 F F F 23 460-479 48 48 48 86 86 86 F F F 24 480-499 53 53 53 92 92 92 F F F 25 500-519 57 57 57 97 97 97 F F F 26 520-539 55 55 55 94 94 94 F F F 27 540-559 57 57 57 97 97 97 F F F 28 560-579 57 57 57 97 97 97 F F F 29 580-599 57 57 57 97 97 97 F F F Max gen: 57, Max flush: 97 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.608924456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.652560629Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.653309107Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.653346098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.653360828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.653376109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414} {"msg":"Crucible 6a34926a-1d19-4a50-a822-b8e89121e165 has session id: 0ec71da6-15f6-4a64-86ec-d48b135c04e4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.653772518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Upstairs opts: Upstairs UUID: 6a34926a-1d19-4a50-a822-b8e89121e165, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.653815929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Crucible stats registered with UUID: 6a34926a-1d19-4a50-a822-b8e89121e165","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.65384071Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683453958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683482518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414} {"msg":"6a34926a-1d19-4a50-a822-b8e89121e165 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.68354317Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.68356301Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683587091Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683601891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683615792Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683638962Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683663533Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683689743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683717764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683744805Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38980)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683766405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63117)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683781725Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40832)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.683795966Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.684530473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.684558714Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.684573655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.684590755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.684604775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.684618585Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685262031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685302292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685332562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685861205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685901036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685923637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685937257Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685951317Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685964968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685977318Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.685990698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[0]R flush_numbers[0..12]: [91, 94, 94, 92, 89, 91, 94, 97, 97, 90, 95, 97]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686044519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[0]R generation[0..12]: [52, 55, 55, 53, 49, 52, 55, 57, 57, 50, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.68606468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686079931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[1]R flush_numbers[0..12]: [91, 94, 94, 92, 89, 91, 94, 97, 97, 90, 95, 97]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686093871Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[1]R generation[0..12]: [52, 55, 55, 53, 49, 52, 55, 57, 57, 50, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686107571Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686121692Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[2]R flush_numbers[0..12]: [91, 94, 94, 92, 89, 91, 94, 97, 97, 90, 95, 97]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686135522Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid"Wait for a query_work_queue command to finish before sending IO :1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[2]R generation[0..12]: [52, 55, 55, 53, 49, 52, 55, 57, 57, 50, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686158112Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686172193Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686185443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686199423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686212643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686225794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"":"downstairs","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686239544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686252624Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"0","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686268195Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"1","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686282085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"client":"2","":"downstairs","session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686295705Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686308106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"6a34926a-1d19-4a50-a822-b8e89121e165 is now active with session: 0ec71da6-15f6-4a64-86ec-d48b135c04e4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686321226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686333426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414,"session_id":"0ec71da6-15f6-4a64-86ec-d48b135c04e4"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:47.686345867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 40960 data: 5 7 9 9 6 6 8 8 7 7 02/30 Write block 396 len 40960 data: 6 7 7 5 4 6 8 10 10 7 03/30 Write block 532 len 36864 data: 6 4 5 5 6 7 6 5 6 04/30 Read block 320 len 16384 05/30 Write block 2 len 32768 data: 3 3 5 4 4 4 4 5 06/30 Write block 252 len 24576 data: 5 4 4 6 6 5 07/30 Write block 491 len 36864 data: 6 4 4 6 4 4 3 3 3 08/30 Write block 441 len 8192 data: 2 2 09/30 Read block 377 len 20480 10/30 Read block 484 len 12288 11/30 Read block 36 len 20480 12/30 Write block 362 len 32768 data: 5 7 6 6 7 6 7 6 13/30 Write block 200 len 16384 data: 5 5 5 4 14/30 Flush 15/30 Write block 319 len 12288 data: 6 4 4 16/30 Read block 327 len 28672 17/30 Write block 376 len 32768 data: 3 7 8 7 6 6 5 4 18/30 Read block 13 len 28672 19/30 Read block 39 len 28672 20/30 Write block 203 len 40960 data: 5 3 4 4 5 6 5 3 4 4 21/30 Read block 454 len 28672 22/30 Write block 396 len 40960 data: 7 8 8 6 5 7 9 11 11 8 23/30 Flush 24/30 Write block 312 len 4096 data: 4 25/30 Read block 279 len 4096 26/30 Write block 213 len 36864 data: 4 3 4 4 3 3 4 5 6 27/30 Write block 473 len 32768 data: 2 2 2 3 4 7 9 9 28/30 Write block 173 len 32768 data: 8 7 6 7 6 6 5 5 29/30 Write block 586 len 28672 data: 4 3 3 6 5 5 5 30/30 Read block 478 len 8192 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:47.706277163Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 98 98 98 F F F 1 020-039 55 55 55 94 94 94 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 53 53 53 92 92 92 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 57 57 57 97 97 97 F F F 8 160-179 57 57 57 97 97 97 T T T 9 180-199 50 50 50 90 90 90 T T T 10 200-219 58 58 58 99 99 99 T T T 11 220-239 57 57 57 97 97 97 T T T 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 55 55 55 94 94 94 F F F 15 300-319 58 58 58 99 99 99 T T T 16 320-339 58 58 58 99 99 99 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 58 58 58 99 99 99 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 58 58 58 99 99 99 F F F 21 420-439 56 56 56 96 96 96 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 48 48 48 86 86 86 T T T 24 480-499 58 58 58 98 98 98 T T T 25 500-519 57 57 57 97 97 97 F F F 26 520-539 58 58 58 98 98 98 F F F 27 540-559 58 58 58 98 98 98 F F F 28 560-579 57 57 57 97 97 97 F F F 29 580-599 57 57 57 97 97 97 T T T Max gen: 58, Max flush: 99 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 98 98 98 F F F 1 020-039 55 55 55 94 94 94 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 53 53 53 92 92 92 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 57 57 57 97 97 97 F F F 8 160-179 59 59 59 100 100 100 F F F 9 180-199 59 59 59 100 100 100 F F F 10 200-219 59 59 59 100 100 100 F F F 11 220-239 59 59 59 100 100 100 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 55 55 55 94 94 94 F F F 15 300-319 59 59 59 100 100 100 F F F 16 320-339 58 58 58 99 99 99 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 58 58 58 99 99 99 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 58 58 58 99 99 99 F F F 21 420-439 56 56 56 96 96 96 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 59 59 59 100 100 100 F F F 24 480-499 59 59 59 100 100 100 F F F 25 500-519 57 57 57 97 97 97 F F F 26 520-539 58 58 58 98 98 98 F F F 27 540-559 58 58 58 98 98 98 F F F 28 560-579 57 57 57 97 97 97 F F F 29 580-599 59 59 59 100 100 100 F F F Max gen: 59, Max flush: 100 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:51.983080846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.025254864Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.025829308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.025877829Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.025893839Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.02590903Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422} {"msg":"Crucible f6bec7e6-af04-420d-a3ba-53ff89efa800 has session id: bc02e110-c87c-4475-9afd-0cc19d35a67e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.026398371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Upstairs opts: Upstairs UUID: f6bec7e6-af04-420d-a3ba-53ff89efa800, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.026444012Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Crucible stats registered with UUID: f6bec7e6-af04-420d-a3ba-53ff89efa800","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.026470473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.055934427Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.055970108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422} {"msg":"f6bec7e6-af04-420d-a3ba-53ff89efa800 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.05604095Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056076241Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056110952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056130902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056154453Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056172463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056189793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056206884Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056224644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056247785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49527)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056280005Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34557)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056319586Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56128)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056363167Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056952572Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.056982232Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.057144276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.057181117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.057218308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.057245639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.057647488Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.05771597Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.057928655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058240662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058279703Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058304494Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058318874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058333274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058346615Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058374755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058389586Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[0]R flush_numbers[0..12]: [98, 94, 94, 92, 89, 91, 94, 97, 100, 100, 100, 100]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058417147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[0]R generation[0..12]: [58, 55, 55, 53, 49, 52, 55, 57, 59, 59, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058431887Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058446387Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[1]R flush_numbers[0..12]: [98, 94, 94, 92, 89, 91, 94, 97, 100, 100, 100, 100]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058460218Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[1]R generation[0..12]: [58, 55, 55, 53, 49, 52, 55, 57, 59, 59, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058477428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058491908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[2]R flush_numbers[0..12]: [98, 94, 94, 92, 89, 91, 94, 97, 100, 100, 100, 100]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058505899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[2]R generation[0..12]: [58, 55, 55, 53, 49, 52, 55, 57, 59, 59, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058519779Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058533919Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.0585474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.05856088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.05857434Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"Wait for a query_work_queue command to finish before sending IO msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058595911Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"":"downstairs","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058609481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058621732Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"0","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058635162Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"1","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058648492Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"client":"2","":"downstairs","session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058662133Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058674573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"f6bec7e6-af04-420d-a3ba-53ff89efa800 is now active with session: bc02e110-c87c-4475-9afd-0cc19d35a67e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058689723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058702243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422,"session_id":"bc02e110-c87c-4475-9afd-0cc19d35a67e"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:52.058714644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 156 len 4096 data: 11 02/30 Write block 557 len 12288 data: 7 5 3 03/30 Read block 166 len 16384 04/30 Write block 284 len 40960 data: 6 7 9 9 7 7 6 7 7 6 05/30 Read block 388 len 36864 06/30 Write block 419 len 40960 data: 4 4 4 5 5 5 6 6 5 5 07/30 Read block 26 len 16384 08/30 Read block 142 len 12288 09/30 Flush 10/30 Write block 222 len 12288 data: 6 7 9 11/30 Write block 510 len 12288 data: 7 6 5 12/30 Read block 122 len 24576 13/30 Write block 544 len 8192 data: 7 5 14/30 Read block 378 len 28672 15/30 Read block 415 len 32768 16/30 Write block 73 len 12288 data: 5 4 3 17/30 Read block 421 len 12288 18/30 Write block 142 len 36864 data: 6 6 6 5 6 5 5 4 4 19/30 Write block 19 len 32768 data: 5 8 7 7 7 8 9 8 20/30 Read block 14 len 32768 21/30 Read block 26 len 28672 22/30 Flush 23/30 Write block 344 len 20480 data: 7 7 8 8 6 24/30 Write block 178 len 24576 data: 7 6 6 3 2 2 25/30 Read block 327 len 24576 26/30 Read block 284 len 40960 27/30 Read block 508 len 24576 28/30 Read block 208 len 28672 29/30 Write block 587 len 4096 data: 4 30/30 Read block 257 len 36864 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:52.078573329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 F F F 1 020-039 60 60 60 102 102 102 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 60 60 60 102 102 102 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 60 60 60 102 102 102 F F F 8 160-179 59 59 59 100 100 100 T T T 9 180-199 59 59 59 100 100 100 T T T 10 200-219 59 59 59 100 100 100 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 60 60 60 101 101 101 F F F 15 300-319 59 59 59 100 100 100 F F F 16 320-339 58 58 58 99 99 99 F F F 17 340-359 57 57 57 97 97 97 T T T 18 360-379 58 58 58 99 99 99 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 60 60 60 101 101 101 F F F 21 420-439 60 60 60 101 101 101 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 59 59 59 100 100 100 F F F 24 480-499 59 59 59 100 100 100 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 58 58 58 98 98 98 F F F 27 540-559 60 60 60 102 102 102 F F F 28 560-579 57 57 57 97 97 97 F F F 29 580-599 59 59 59 100 100 100 T T T Max gen: 60, Max flush: 102 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 F F F 1 020-039 60 60 60 102 102 102 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 60 60 60 102 102 102 F F F 4 080-099 49 49 49 89 89 89 F F F 5 100-119 52 52 52 91 91 91 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 60 60 60 102 102 102 F F F 8 160-179 61 61 61 103 103 103 F F F 9 180-199 61 61 61 103 103 103 F F F 10 200-219 59 59 59 100 100 100 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 60 60 60 101 101 101 F F F 15 300-319 59 59 59 100 100 100 F F F 16 320-339 58 58 58 99 99 99 F F F 17 340-359 61 61 61 103 103 103 F F F 18 360-379 58 58 58 99 99 99 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 60 60 60 101 101 101 F F F 21 420-439 60 60 60 101 101 101 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 59 59 59 100 100 100 F F F 24 480-499 59 59 59 100 100 100 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 58 58 58 98 98 98 F F F 27 540-559 60 60 60 102 102 102 F F F 28 560-579 57 57 57 97 97 97 F F F 29 580-599 61 61 61 103 103 103 F F F Max gen: 61, Max flush: 103 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.334516172Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:56.375712037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.376088156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.406631176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.40719103Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.407227341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.407241551Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.407256031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430} {"msg":"Crucible a23dac2f-b2d5-4c97-bc3d-167b7bad9d7e has session id: 6aaeb028-04e0-4abc-9282-9cafe6cf3b78","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.407658351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Upstairs opts: Upstairs UUID: a23dac2f-b2d5-4c97-bc3d-167b7bad9d7e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.407703752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Crucible stats registered with UUID: a23dac2f-b2d5-4c97-bc3d-167b7bad9d7e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.407727873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437511354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437536785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430} {"msg":"a23dac2f-b2d5-4c97-bc3d-167b7bad9d7e active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437613887Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437631927Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437669518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437694169Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437709389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.43772671Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.43774246Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437766181Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437791321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437808712Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_connection connected from Ok(127.0.0.1:42685)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.437961265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_connection connected from Ok(127.0.0.1:61043)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.438004076Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_connection connected from Ok(127.0.0.1:53206)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.438034307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.438785525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.438825376Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.438856777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.438887017Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.438948369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.43898382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.439481652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.439520562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.439556363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440085576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440111197Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440127587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440140537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440155228Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440170038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440182638Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440196899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[0]R flush_numbers[0..12]: [102, 102, 94, 102, 89, 91, 94, 102, 103, 103, 100, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440211409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[0]R generation[0..12]: [60, 60, 55, 60, 49, 52, 55, 60, 61, 61, 59, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440225599Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.44024066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[1]R flush_numbers[0..12]: [102, 102, 94, 102, 89, 91, 94, 102, 103, 103, 100, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.44025556Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[1]R generation[0..12]: [60, 60, 55, 60, 49, 52, 55, 60, 61, 61, 59, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440269531Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440287531Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[2]R flush_numbers[0..12]: [102, 102, 94, 102, 89, 91, 94, 102, 103, 103, 100, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440302931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[2]R generation[0..12]: [60, 60, 55, 60, 49, 52, 55, 60, 61, 61, 59, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440326872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440341972Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440355993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440369983Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440384003Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440398034Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"":"downstairs","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440412554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:09:56.440425704Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"0","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440441184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"1","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440455645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"client":"2","":"downstairs","session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440470065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440483476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"a23dac2f-b2d5-4c97-bc3d-167b7bad9d7e is now active with session: 6aaeb028-04e0-4abc-9282-9cafe6cf3b78","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440497206Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440512536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430,"session_id":"6aaeb028-04e0-4abc-9282-9cafe6cf3b78"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:56.440526037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 107 len 12288 data: 5 5 5 02/30 Read block 298 len 32768 03/30 Read block 173 len 28672 04/30 Read block 329 len 36864 05/30 Write block 196 len 36864 data: 4 6 6 6 6 6 6 6 4 06/30 Read block 407 len 16384 07/30 Write block 594 len 12288 data: 4 4 4 08/30 Write block 520 len 4096 data: 4 09/30 Write block 297 len 4096 data: 10 10/30 Read block 272 len 4096 11/30 Read block 1 len 4096 12/30 Read block 568 len 8192 13/30 Read block 589 len 24576 14/30 Write block 55 len 28672 data: 8 8 6 5 5 5 6 15/30 Flush 16/30 Write block 334 len 16384 data: 6 6 5 5 17/30 Read block 362 len 4096 18/30 Read block 419 len 8192 19/30 Read block 476 len 8192 20/30 Write block 92 len 16384 data: 4 4 4 6 21/30 Write block 412 len 32768 data: 8 7 7 5 4 5 4 5 22/30 Read block 145 len 20480 23/30 Write block 570 len 40960 data: 6 5 5 6 7 6 6 5 6 7 24/30 Flush 25/30 Write block 459 len 40960 data: 5 3 3 4 5 4 3 3 3 4 26/30 Read block 428 len 24576 27/30 Read block 412 len 8192 28/30 Read block 358 len 36864 29/30 Read block 358 len 24576 30/30 Write block 19 len 20480 data: 6 9 8 8 8 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:56.45989412Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 T T T 1 020-039 60 60 60 102 102 102 T T T 2 040-059 62 62 62 104 104 104 F F F 3 060-079 62 62 62 104 104 104 F F F 4 080-099 62 62 62 105 105 105 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 60 60 60 102 102 102 F F F 8 160-179 61 61 61 103 103 103 F F F 9 180-199 62 62 62 104 104 104 F F F 10 200-219 62 62 62 104 104 104 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 62 62 62 104 104 104 F F F 15 300-319 59 59 59 100 100 100 F F F 16 320-339 62 62 62 105 105 105 F F F 17 340-359 61 61 61 103 103 103 F F F 18 360-379 58 58 58 99 99 99 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 60 60 60 101 101 101 F F F 22 440-459 58 58 58 98 98 98 T T T 23 460-479 59 59 59 100 100 100 T T T 24 480-499 59 59 59 100 100 100 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 62 62 62 104 104 104 F F F 27 540-559 60 60 60 102 102 102 F F F 28 560-579 62 62 62 105 105 105 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 62, Max flush: 105 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 106 106 106 F F F 1 020-039 63 63 63 106 106 106 F F F 2 040-059 62 62 62 104 104 104 F F F 3 060-079 62 62 62 104 104 104 F F F 4 080-099 62 62 62 105 105 105 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 60 60 60 102 102 102 F F F 8 160-179 61 61 61 103 103 103 F F F 9 180-199 62 62 62 104 104 104 F F F 10 200-219 62 62 62 104 104 104 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 62 62 62 104 104 104 F F F 15 300-319 59 59 59 100 100 100 F F F 16 320-339 62 62 62 105 105 105 F F F 17 340-359 61 61 61 103 103 103 F F F 18 360-379 58 58 58 99 99 99 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 60 60 60 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 63 63 63 106 106 106 F F F 24 480-499 59 59 59 100 100 100 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 62 62 62 104 104 104 F F F 27 540-559 60 60 60 102 102 102 F F F 28 560-579 62 62 62 105 105 105 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 63, Max flush: 106 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.744745025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.787200069Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.787790733Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.787826184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.787840754Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.787860045Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438} {"msg":"Crucible 67c7f79b-446d-45f6-af3d-52707ed3343b has session id: 8688d299-adfb-4b0e-bee4-8d4faae2851f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.788248124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Upstairs opts: Upstairs UUID: 67c7f79b-446d-45f6-af3d-52707ed3343b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.788292835Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Crucible stats registered with UUID: 67c7f79b-446d-45f6-af3d-52707ed3343b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.788319146Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.817940664Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.817975065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438} {"msg":"67c7f79b-446d-45f6-af3d-52707ed3343b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.817996185Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818019106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818047666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818065827Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818082937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818099738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818116438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818133019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818151719Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.81818737Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40092)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818297252Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51716)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818343073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49180)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.818459496Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819089321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819119442Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819135503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819149483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819163063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819184174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819673265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.81986791Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.819983113Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820220648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820257519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.82027774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820343921Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820363552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820377822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820569167Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820588127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[0]R flush_numbers[0..12]: [106, 106, 104, 104, 105, 104, 94, 102, 103, 104, 104, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820602487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[0]R generation[0..12]: [63, 63, 62, 62, 62, 62, 55, 60, 61, 62, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820616418Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820630618Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[1]R flush_numbers[0..12]: [106, 106, 104, 104, 105, 104, 94, 102, 103, 104, 104, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820644548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[1]R generation[0..12]: [63, 63, 62, 62, 62, 62, 55, 60, 61, 62, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820658339Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820672369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[2]R flush_numbers[0..12]: [106, 106, 104, 104, 105, 104, 94, 102, 103, 104, 104, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.82068616Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[2]R generation[0..12]: [63, 63, 62, 62, 62, 62, 55, 60, 61, 62, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.82069989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.82071398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.82073017Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820744001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820757381Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:10:00.820770682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"":"downstairs","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820789782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820801952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"0","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820815293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"1","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820828423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"client":"2","":"downstairs","session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820841643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820854073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"67c7f79b-446d-45f6-af3d-52707ed3343b is now active with session: 8688d299-adfb-4b0e-bee4-8d4faae2851f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820867024Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820879144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438,"session_id":"8688d299-adfb-4b0e-bee4-8d4faae2851f"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:00.820891625Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 316 len 8192 data: 7 7 02/30 Read block 362 len 40960 03/30 Write block 349 len 16384 data: 6 5 4 5 04/30 Flush 05/30 Write block 217 len 24576 data: 4 4 5 6 7 7 06/30 Read block 505 len 32768 07/30 Write block 58 len 28672 data: 6 6 6 7 6 6 7 08/30 Write block 552 len 8192 data: 7 7 09/30 Write block 368 len 12288 data: 8 7 5 10/30 Write block 65 len 24576 data: 8 9 10 7 7 7 11/30 Read block 593 len 24576 12/30 Read block 534 len 12288 13/30 Write block 522 len 8192 data: 4 4 14/30 Read block 252 len 20480 15/30 Write block 21 len 24576 data: 9 9 9 9 10 9 16/30 Read block 546 len 12288 17/30 Write block 338 len 40960 data: 6 6 6 6 5 6 8 8 9 9 18/30 Read block 19 len 40960 19/30 Write block 28 len 40960 data: 7 6 7 5 5 5 5 4 5 7 20/30 Read block 451 len 20480 21/30 Write block 43 len 20480 data: 9 7 7 8 6 22/30 Flush 23/30 Write block 169 len 4096 data: 4 24/30 Read block 294 len 8192 25/30 Read block 136 len 8192 26/30 Read block 359 len 12288 27/30 Write block 152 len 20480 data: 4 8 10 11 12 28/30 Write block 569 len 12288 data: 6 7 6 29/30 Read block 317 len 8192 30/30 Write block 321 len 40960 data: 5 4 4 4 4 5 5 4 4 4 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:00.840084343Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 106 106 106 F F F 1 020-039 64 64 64 108 108 108 F F F 2 040-059 64 64 64 108 108 108 F F F 3 060-079 64 64 64 108 108 108 F F F 4 080-099 62 62 62 105 105 105 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 60 60 60 102 102 102 T T T 8 160-179 61 61 61 103 103 103 T T T 9 180-199 62 62 62 104 104 104 F F F 10 200-219 64 64 64 108 108 108 F F F 11 220-239 64 64 64 108 108 108 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 62 62 62 104 104 104 F F F 15 300-319 64 64 64 107 107 107 F F F 16 320-339 64 64 64 108 108 108 T T T 17 340-359 64 64 64 108 108 108 F F F 18 360-379 64 64 64 108 108 108 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 60 60 60 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 63 63 63 106 106 106 F F F 24 480-499 59 59 59 100 100 100 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 64 64 64 108 108 108 F F F 28 560-579 62 62 62 105 105 105 T T T 29 580-599 62 62 62 104 104 104 F F F Max gen: 64, Max flush: 108 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 106 106 106 F F F 1 020-039 64 64 64 108 108 108 F F F 2 040-059 64 64 64 108 108 108 F F F 3 060-079 64 64 64 108 108 108 F F F 4 080-099 62 62 62 105 105 105 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 65 65 65 109 109 109 F F F 9 180-199 62 62 62 104 104 104 F F F 10 200-219 64 64 64 108 108 108 F F F 11 220-239 64 64 64 108 108 108 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 62 62 62 104 104 104 F F F 15 300-319 64 64 64 107 107 107 F F F 16 320-339 65 65 65 109 109 109 F F F 17 340-359 64 64 64 108 108 108 F F F 18 360-379 64 64 64 108 108 108 F F F 19 380-399 58 58 58 99 99 99 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 60 60 60 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 63 63 63 106 106 106 F F F 24 480-499 59 59 59 100 100 100 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 64 64 64 108 108 108 F F F 28 560-579 65 65 65 109 109 109 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 65, Max flush: 109 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.093117238Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:05.134765693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.135225955Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.166182664Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.166802199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.16683704Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.166851131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.166865421Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446} {"msg":"Crucible bee3bb74-20ba-4b48-8a1b-96ee9815bfca has session id: dd124eb7-534f-45d2-827f-dc4efd8fd300","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.16726566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Upstairs opts: Upstairs UUID: bee3bb74-20ba-4b48-8a1b-96ee9815bfca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.167308891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Crucible stats registered with UUID: bee3bb74-20ba-4b48-8a1b-96ee9815bfca","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.167326832Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.196887438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.19693205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446} {"msg":"bee3bb74-20ba-4b48-8a1b-96ee9815bfca active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.196999531Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197035422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197069703Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197089053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197106564Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197123394Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197140395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197156885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197174515Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197191946Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_connection connected from Ok(127.0.0.1:54913)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.1973728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_connection connected from Ok(127.0.0.1:43388)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197472203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_connection connected from Ok(127.0.0.1:55154)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.197508943Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198080207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198107398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198122748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198136698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198150189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198163529Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198843435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198873386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.198889086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199422989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.19946345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199492071Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199513391Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199541402Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199567293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199591013Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199614604Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[0]R flush_numbers[0..12]: [106, 108, 108, 108, 105, 104, 94, 109, 109, 104, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199638814Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[0]R generation[0..12]: [63, 64, 64, 64, 62, 62, 55, 65, 65, 62, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199664685Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199692776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[1]R flush_numbers[0..12]: [106, 108, 108, 108, 105, 104, 94, 109, 109, 104, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199718716Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[1]R generation[0..12]: [63, 64, 64, 64, 62, 62, 55, 65, 65, 62, 64, 64]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:10:05.199746477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199783798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[2]R flush_numbers[0..12]: [106, 108, 108, 108, 105, 104, 94, 109, 109, 104, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199815928Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[2]R generation[0..12]: [63, 64, 64, 64, 62, 62, 55, 65, 65, 62, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199846489Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.19987908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199911661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "crucible","level":30,"time":"2024-12-09T15:10:05.199946492Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199970022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199984363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"":"downstairs","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.199998253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200010553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"0","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200024284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"1","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200037764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"client":"2","":"downstairs","session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200051154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200063645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"bee3bb74-20ba-4b48-8a1b-96ee9815bfca is now active with session: dd124eb7-534f-45d2-827f-dc4efd8fd300","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200076715Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200088925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446,"session_id":"dd124eb7-534f-45d2-827f-dc4efd8fd300"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:05.200101305Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 8192 data: 4 4 02/30 Flush 03/30 Write block 454 len 12288 data: 4 5 4 04/30 Read block 211 len 4096 05/30 Write block 471 len 12288 data: 5 3 3 06/30 Flush 07/30 Write block 495 len 40960 data: 5 5 4 4 5 5 4 5 5 4 08/30 Write block 327 len 20480 data: 6 5 5 5 5 09/30 Read block 580 len 36864 10/30 Read block 334 len 36864 11/30 Read block 107 len 8192 12/30 Read block 347 len 28672 13/30 Write block 14 len 12288 data: 5 6 8 14/30 Read block 592 len 20480 15/30 Read block 45 len 40960 16/30 Flush 17/30 Write block 22 len 4096 data: 10 18/30 Write block 376 len 36864 data: 4 8 9 8 7 7 6 5 5 19/30 Read block 149 len 36864 20/30 Write block 426 len 16384 data: 7 6 6 6 21/30 Flush 22/30 Flush 23/30 Flush 24/30 Write block 191 len 16384 data: 2 3 5 6 25/30 Flush 26/30 Write block 181 len 12288 data: 4 3 3 27/30 Read block 151 len 36864 28/30 Write block 451 len 24576 data: 3 3 4 5 6 5 29/30 Write block 481 len 8192 data: 12 9 30/30 Read block 288 len 32768 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:05.218836223Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 F F F 1 020-039 66 66 66 113 113 113 F F F 2 040-059 64 64 64 108 108 108 F F F 3 060-079 64 64 64 108 108 108 F F F 4 080-099 62 62 62 105 105 105 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 65 65 65 109 109 109 F F F 9 180-199 66 66 66 116 116 116 T T T 10 200-219 64 64 64 108 108 108 F F F 11 220-239 64 64 64 108 108 108 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 62 62 62 104 104 104 F F F 15 300-319 64 64 64 107 107 107 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 64 64 64 108 108 108 F F F 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 66 66 66 113 113 113 F F F 22 440-459 66 66 66 111 111 111 T T T 23 460-479 66 66 66 111 111 111 F F F 24 480-499 66 66 66 112 112 112 T T T 25 500-519 66 66 66 112 112 112 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 64 64 64 108 108 108 F F F 28 560-579 65 65 65 109 109 109 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 66, Max flush: 116 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 F F F 1 020-039 66 66 66 113 113 113 F F F 2 040-059 64 64 64 108 108 108 F F F 3 060-079 64 64 64 108 108 108 F F F 4 080-099 62 62 62 105 105 105 F F F 5 100-119 62 62 62 104 104 104 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 65 65 65 109 109 109 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 64 64 64 108 108 108 F F F 11 220-239 64 64 64 108 108 108 F F F 12 240-259 58 58 58 98 98 98 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 62 62 62 104 104 104 F F F 15 300-319 64 64 64 107 107 107 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 64 64 64 108 108 108 F F F 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 66 66 66 113 113 113 F F F 22 440-459 67 67 67 117 117 117 F F F 23 460-479 66 66 66 111 111 111 F F F 24 480-499 67 67 67 117 117 117 F F F 25 500-519 66 66 66 112 112 112 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 64 64 64 108 108 108 F F F 28 560-579 65 65 65 109 109 109 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 67, Max flush: 117 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.49539808Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:09.536350279Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.536715038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.566981112Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.567546185Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.567580476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.567594546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.567609967Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454} {"msg":"Crucible 7f140937-959d-46f3-b7a1-232d4e5d830d has session id: 44570db5-1222-4f1d-8e31-02d129174ffe","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.568069967Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Upstairs opts: Upstairs UUID: 7f140937-959d-46f3-b7a1-232d4e5d830d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.568113859Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Crucible stats registered with UUID: 7f140937-959d-46f3-b7a1-232d4e5d830d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.568144199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.597731367Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.597762727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454} {"msg":"7f140937-959d-46f3-b7a1-232d4e5d830d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.597838349Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.59786074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.59789142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.597909951Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.597934591Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.597966892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598001443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598036164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598071765Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598107085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:41636)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.59831109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:65159)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598359722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:43496)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598400043Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598816033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598849393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.598979626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.599050568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.599088419Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.599112269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.599465748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.599496269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.599615752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600089123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600119634Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600134514Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600147374Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600161214Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600174745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600187315Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600205006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[0]R flush_numbers[0..12]: [112, 113, 108, 108, 105, 104, 94, 109, 109, 117, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600219576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[0]R generation[0..12]: [66, 66, 64, 64, 62, 62, 55, 65, 65, 67, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600233746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600248097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[1]R flush_numbers[0..12]: [112, 113, 108, 108, 105, 104, 94, 109, 109, 117, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600262057Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[1]R generation[0..12]: [66, 66, 64, 64, 62, 62, 55, 65, 65, 67, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600275757Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600289828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[2]R flush_numbers[0..12]: [112, 113, 108, 108, 105, 104, 94, 109, 109, 117, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600303968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[2]R generation[0..12]: [66, 66, 64, 64, 62, 62, 55, 65, 65, 67, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600317998Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600331989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600345429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600358789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.60037198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.60038548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid"Wait for a query_work_queue command to finish before sending IO :1454,"":"downstairs","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600410371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600422761Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"0","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600436341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"1","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600449772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"client":"2","":"downstairs","session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600463222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600475722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"7f140937-959d-46f3-b7a1-232d4e5d830d is now active with session: 44570db5-1222-4f1d-8e31-02d129174ffe","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600488713Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600500903Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454,"session_id":"44570db5-1222-4f1d-8e31-02d129174ffe"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:09.600513393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 109 len 8192 data: 6 5 02/30 Write block 422 len 16384 data: 6 6 6 7 03/30 Write block 3 len 20480 data: 4 6 5 5 5 04/30 Read block 519 len 16384 05/30 Read block 411 len 20480 06/30 Read block 372 len 32768 07/30 Write block 479 len 16384 data: 10 10 13 10 08/30 Read block 416 len 8192 09/30 Write block 527 len 28672 data: 3 3 3 4 5 7 5 10/30 Read block 2 len 24576 11/30 Write block 88 len 24576 data: 4 4 5 4 5 5 12/30 Read block 297 len 32768 13/30 Read block 528 len 36864 14/30 Write block 99 len 8192 data: 9 11 15/30 Read block 77 len 32768 16/30 Read block 421 len 4096 17/30 Write block 123 len 32768 data: 4 3 3 3 5 4 4 4 18/30 Write block 298 len 8192 data: 7 4 19/30 Read block 136 len 40960 20/30 Read block 39 len 4096 21/30 Read block 505 len 20480 22/30 Read block 551 len 16384 23/30 Read block 25 len 8192 24/30 Read block 177 len 36864 25/30 Write block 233 len 12288 data: 9 7 5 26/30 Read block 220 len 16384 27/30 Write block 233 len 32768 data: 10 8 6 5 6 6 6 4 28/30 Write block 209 len 36864 data: 6 4 5 5 5 4 5 5 5 29/30 Write block 346 len 8192 data: 10 10 30/30 Write block 58 len 36864 data: 7 7 7 8 7 7 8 9 10 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:09.618861942Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 112 112 112 T T T 1 020-039 66 66 66 113 113 113 F F F 2 040-059 64 64 64 108 108 108 T T T 3 060-079 64 64 64 108 108 108 T T T 4 080-099 62 62 62 105 105 105 T T T 5 100-119 62 62 62 104 104 104 T T T 6 120-139 55 55 55 94 94 94 T T T 7 140-159 65 65 65 109 109 109 F F F 8 160-179 65 65 65 109 109 109 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 64 64 64 108 108 108 T T T 11 220-239 64 64 64 108 108 108 T T T 12 240-259 58 58 58 98 98 98 T T T 13 260-279 48 48 48 88 88 88 F F F 14 280-299 62 62 62 104 104 104 T T T 15 300-319 64 64 64 107 107 107 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 64 64 64 108 108 108 T T T 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 66 66 66 113 113 113 T T T 22 440-459 67 67 67 117 117 117 F F F 23 460-479 66 66 66 111 111 111 T T T 24 480-499 67 67 67 117 117 117 T T T 25 500-519 66 66 66 112 112 112 F F F 26 520-539 64 64 64 108 108 108 T T T 27 540-559 64 64 64 108 108 108 F F F 28 560-579 65 65 65 109 109 109 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 67, Max flush: 117 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 118 118 118 F F F 1 020-039 66 66 66 113 113 113 F F F 2 040-059 68 68 68 118 118 118 F F F 3 060-079 68 68 68 118 118 118 F F F 4 080-099 68 68 68 118 118 118 F F F 5 100-119 68 68 68 118 118 118 F F F 6 120-139 68 68 68 118 118 118 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 65 65 65 109 109 109 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 68 68 68 118 118 118 F F F 12 240-259 68 68 68 118 118 118 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 68 68 68 118 118 118 F F F 15 300-319 64 64 64 107 107 107 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 68 68 68 118 118 118 F F F 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 67 67 67 117 117 117 F F F 23 460-479 68 68 68 118 118 118 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 66 66 66 112 112 112 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 64 64 64 108 108 108 F F F 28 560-579 65 65 65 109 109 109 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 68, Max flush: 118 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:13.960459483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:14.001952755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.002390675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.034000431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.034593915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.034629886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.034645166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.034660147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462} {"msg":"Crucible cc8aaea0-c4af-449c-bfdc-e5b2c595306a has session id: 50124ffd-937e-4add-86c9-d6fd3103456b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.035077097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Upstairs opts: Upstairs UUID: cc8aaea0-c4af-449c-bfdc-e5b2c595306a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.035113207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Crucible stats registered with UUID: cc8aaea0-c4af-449c-bfdc-e5b2c595306a","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.035130088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066042807Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066077638Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462} {"msg":"cc8aaea0-c4af-449c-bfdc-e5b2c595306a active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066122079Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066139549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.06616481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.06617941Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.06619323Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066207211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066221291Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066291103Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066325784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066356344Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46569)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066392575Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44811)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066427426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43599)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066551699Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.066985789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067043711Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067074161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067103752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067139413Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067169794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067682936Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067712727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.067728987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.06826501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068305331Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068327771Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068345052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068359532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068373822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068386133Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068399943Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[0]R flush_numbers[0..12]: [118, 113, 118, 118, 118, 118, 118, 109, 109, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068414093Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[0]R generation[0..12]: [68, 66, 68, 68, 68, 68, 68, 65, 65, 67, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068427934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068442154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[1]R flush_numbers[0..12]: [118, 113, 118, 118, 118, 118, 118, 109, 109, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068456684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[1]R generation[0..12]: [68, 66, 68, 68, 68, 68, 68, 65, 65, 67, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068470615Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068484715Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[2]R flush_numbers[0..12]: [118, 113, 118, 118, 118, 118, 118, 109, 109, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068498836Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[2]R generation[0..12]: [68, 66, 68, 68, 68, 68, 68, 65, 65, 67, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068512526Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068540677Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Max found gen is 69","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068570387Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Generation requested: 70 >= found:69","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068594788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068617418Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068632949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"":"downstairs","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068648599Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068671429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"0","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2024-12-09T15:10:14.06869354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"1","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.06870764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"client":"2","":"downstairs","session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068721211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068733821Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"cc8aaea0-c4af-449c-bfdc-e5b2c595306a is now active with session: 50124ffd-937e-4add-86c9-d6fd3103456b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068746891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068759072Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462,"session_id":"50124ffd-937e-4add-86c9-d6fd3103456b"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:14.068771372Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 8192 data: 5 7 02/30 Read block 536 len 20480 03/30 Write block 92 len 40960 data: 6 6 5 7 6 6 8 10 12 11 04/30 Write block 428 len 12288 data: 7 7 6 05/30 Read block 51 len 4096 06/30 Write block 222 len 8192 data: 8 8 07/30 Read block 155 len 8192 08/30 Write block 503 len 36864 data: 6 5 5 6 7 7 6 8 7 09/30 Write block 554 len 20480 data: 9 7 7 8 6 10/30 Read block 512 len 40960 11/30 Write block 54 len 16384 data: 7 9 9 7 12/30 Read block 47 len 4096 13/30 Flush 14/30 Write block 178 len 32768 data: 8 7 7 5 4 4 2 3 15/30 Write block 515 len 12288 data: 7 6 6 16/30 Write block 501 len 4096 data: 5 17/30 Write block 111 len 28672 data: 4 4 5 4 4 4 4 18/30 Flush 19/30 Write block 161 len 8192 data: 11 11 20/30 Write block 344 len 36864 data: 9 9 11 11 7 7 6 5 6 21/30 Read block 346 len 24576 22/30 Read block 485 len 8192 23/30 Flush 24/30 Write block 73 len 36864 data: 6 5 4 3 6 6 5 4 5 25/30 Read block 109 len 32768 26/30 Write block 13 len 28672 data: 4 6 7 9 8 7 7 27/30 Write block 62 len 32768 data: 8 8 9 10 11 11 8 8 28/30 Write block 557 len 4096 data: 9 29/30 Write block 557 len 16384 data: 10 7 4 2 30/30 Write block 126 len 8192 data: 4 6 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 3 10 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:14.086315481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 118 118 118 T T T 1 020-039 66 66 66 113 113 113 F F F 2 040-059 70 70 70 119 119 119 F F F 3 060-079 68 68 68 118 118 118 T T T 4 080-099 70 70 70 119 119 119 T T T 5 100-119 70 70 70 120 120 120 F F F 6 120-139 68 68 68 118 118 118 T T T 7 140-159 65 65 65 109 109 109 F F F 8 160-179 70 70 70 121 121 121 F F F 9 180-199 70 70 70 120 120 120 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 70 70 70 119 119 119 F F F 12 240-259 68 68 68 118 118 118 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 68 68 68 118 118 118 F F F 15 300-319 64 64 64 107 107 107 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 70 70 70 121 121 121 F F F 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 70 70 70 119 119 119 F F F 22 440-459 67 67 67 117 117 117 F F F 23 460-479 68 68 68 118 118 118 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 70 70 70 119 119 119 T T T 28 560-579 65 65 65 109 109 109 T T T 29 580-599 62 62 62 104 104 104 F F F Max gen: 70, Max flush: 121 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 66 66 66 113 113 113 F F F 2 040-059 70 70 70 119 119 119 F F F 3 060-079 71 71 71 122 122 122 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 70 70 70 120 120 120 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 70 70 70 121 121 121 F F F 9 180-199 70 70 70 120 120 120 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 70 70 70 119 119 119 F F F 12 240-259 68 68 68 118 118 118 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 68 68 68 118 118 118 F F F 15 300-319 64 64 64 107 107 107 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 70 70 70 121 121 121 F F F 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 70 70 70 119 119 119 F F F 22 440-459 67 67 67 117 117 117 F F F 23 460-479 68 68 68 118 118 118 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 71 71 71 122 122 122 F F F 28 560-579 71 71 71 122 122 122 F F F 29 580-599 62 62 62 104 104 104 F F F Max gen: 71, Max flush: 122 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.384128087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:18.425453125Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.425811063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.456245901Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.456800834Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.456833175Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.456846795Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.456861055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470} {"msg":"Crucible b9abda03-7ba5-4e66-86bd-ce21de7fb282 has session id: e1e9be23-76f4-44a8-860b-af391d83bcf3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.457230544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Upstairs opts: Upstairs UUID: b9abda03-7ba5-4e66-86bd-ce21de7fb282, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.457283895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Crucible stats registered with UUID: b9abda03-7ba5-4e66-86bd-ce21de7fb282","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.457305376Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.48719277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487223851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470} {"msg":"b9abda03-7ba5-4e66-86bd-ce21de7fb282 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487294613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487317523Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487349404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487368545Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487385985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487413135Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487446626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487478477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487512878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487548179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54695)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487671422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43339)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487727583Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43431)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.487760564Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.488319497Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.488361548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.488393569Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.488465231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.488485481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.488499872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.488988203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489033794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489141287Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489486885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489526356Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489554287Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489577247Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489595918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489610368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489631499Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489645639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[0]R flush_numbers[0..12]: [122, 113, 119, 122, 122, 120, 122, 109, 121, 120, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.48968648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[0]R generation[0..12]: [71, 66, 70, 71, 71, 70, 71, 65, 70, 70, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.48970404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489718561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[1]R flush_numbers[0..12]: [122, 113, 119, 122, 122, 120, 122, 109, 121, 120, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489732481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[1]R generation[0..12]: [71, 66, 70, 71, 71, 70, 71, 65, 70, 70, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489746141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489760252Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[2]R flush_numbers[0..12]: [122, 113, 119, 122, 122, 120, 122, 109, 121, 120, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489774172Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[2]R generation[0..12]: [71, 66, 70, 71, 71, 70, 71, 65, 70, 70, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489787862Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489802163Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489815373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489828673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489845224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489911685Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"":"downstairs","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489931076Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489943666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"0","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489957286Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"1","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489970467Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"client":"2","":"downstairs","session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489983947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.489996307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"b9abda03-7ba5-4e66-86bd-ce21de7fb282 is now active with session: e1e9be23-76f4-44a8-860b-af391d83bcf3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.490009198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.490021408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470,"session_id":"e1e9be23-76f4-44a8-860b-af391d83bcf3"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:18.490033688Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 25 len 8192 data: 11 10 02/30 Read block 87 len 16384 03/30 Flush 04/30 Write block 584 len 16384 data: 6 5 5 5 05/30 Write block 258 len 32768 data: 7 6 5 4 4 3 3 3 06/30 Read block 108 len 24576 07/30 Read block 203 len 20480 08/30 Write block 191 len 28672 data: 3 4 6 7 6 6 8 09/30 Flush 10/30 Write block 548 len 12288 data: 4 4 5 11/30 Read block 137 len 20480 12/30 Read block 547 len 32768 13/30 Read block 486 len 36864 14/30 Read block 67 len 20480 15/30 Read block 52 len 20480 16/30 Flush 17/30 Write block 460 len 32768 data: 4 4 5 6 5 4 4 4 18/30 Read block 301 len 4096 19/30 Write block 53 len 36864 data: 6 8 10 10 8 8 8 8 9 20/30 Read block 177 len 24576 21/30 Read block 506 len 24576 22/30 Write block 275 len 12288 data: 7 5 4 23/30 Write block 356 len 8192 data: 6 5 24/30 Write block 306 len 8192 data: 7 7 25/30 Read block 72 len 36864 26/30 Read block 231 len 8192 27/30 Read block 77 len 24576 28/30 Read block 435 len 8192 29/30 Read block 359 len 12288 30/30 Write block 580 len 36864 data: 7 8 7 8 7 6 6 6 4 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:18.509581016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 72 72 72 123 123 123 F F F 2 040-059 70 70 70 119 119 119 T T T 3 060-079 71 71 71 122 122 122 T T T 4 080-099 71 71 71 122 122 122 F F F 5 100-119 70 70 70 120 120 120 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 70 70 70 121 121 121 F F F 9 180-199 72 72 72 124 124 124 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 70 70 70 119 119 119 F F F 12 240-259 72 72 72 124 124 124 F F F 13 260-279 72 72 72 124 124 124 T T T 14 280-299 68 68 68 118 118 118 F F F 15 300-319 64 64 64 107 107 107 T T T 16 320-339 66 66 66 112 112 112 F F F 17 340-359 70 70 70 121 121 121 T T T 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 70 70 70 119 119 119 F F F 22 440-459 67 67 67 117 117 117 F F F 23 460-479 68 68 68 118 118 118 T T T 24 480-499 68 68 68 118 118 118 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 125 125 125 F F F 28 560-579 71 71 71 122 122 122 F F F 29 580-599 72 72 72 124 124 124 T T T Max gen: 72, Max flush: 125 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 72 72 72 123 123 123 F F F 2 040-059 73 73 73 126 126 126 F F F 3 060-079 73 73 73 126 126 126 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 70 70 70 120 120 120 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 70 70 70 121 121 121 F F F 9 180-199 72 72 72 124 124 124 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 70 70 70 119 119 119 F F F 12 240-259 72 72 72 124 124 124 F F F 13 260-279 73 73 73 126 126 126 F F F 14 280-299 68 68 68 118 118 118 F F F 15 300-319 73 73 73 126 126 126 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 66 66 66 113 113 113 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 62 62 62 105 105 105 F F F 21 420-439 70 70 70 119 119 119 F F F 22 440-459 67 67 67 117 117 117 F F F 23 460-479 73 73 73 126 126 126 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 125 125 125 F F F 28 560-579 71 71 71 122 122 122 F F F 29 580-599 73 73 73 126 126 126 F F F Max gen: 73, Max flush: 126 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.808629649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.852198601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.852841026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.852876807Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.852892098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.852906488Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478} {"msg":"Crucible 0a574d3e-3512-4a12-87e1-ee2439928cf0 has session id: 2b65f031-b6f3-48fc-9839-18f70e214890","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.853304308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Upstairs opts: Upstairs UUID: 0a574d3e-3512-4a12-87e1-ee2439928cf0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.853351318Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Crucible stats registered with UUID: 0a574d3e-3512-4a12-87e1-ee2439928cf0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.853382249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.883819447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.883864258Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478} {"msg":"0a574d3e-3512-4a12-87e1-ee2439928cf0 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.88393553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.88397059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884007731Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884042142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884076033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884111024Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884146885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884183026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884220456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884295128Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_connection connected from Ok(127.0.0.1:55649)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884333569Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_connection connected from Ok(127.0.0.1:37845)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.88436606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_connection connected from Ok(127.0.0.1:40034)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884400871Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884884752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.884942754Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.885036766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.885070237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.885101648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.885129208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.885536078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.885577939Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.885810465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886101272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886129232Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886145233Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886227284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886247485Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886261605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886316726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886340377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[0]R flush_numbers[0..12]: [122, 123, 126, 126, 122, 120, 122, 109, 121, 124, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886355987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[0]R generation[0..12]: [71, 72, 73, 73, 71, 70, 71, 65, 70, 72, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886370398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886385568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[1]R flush_numbers[0..12]: [122, 123, 126, 126, 122, 120, 122, 109, 121, 124, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886399508Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[1]R generation[0..12]: [71, 72, 73, 73, 71, 70, 71, 65, 70, 72, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886412969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886427029Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[2]R flush_numbers[0..12]: [122, 123, 126, 126, 122, 120, 122, 109, 121, 124, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.8864409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[2]R generation[0..12]: [71, 72, 73, 73, 71, 70, 71, 65, 70, 72, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.88645561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.88647069Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.88648403Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886497621Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886510941Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886524431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"":"downstairs","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886540032Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886563212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"0","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886576723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"1","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886589933Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"client":"2","":"downstairs","session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886603184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886615424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"0a574d3e-3512-4a12-87e1-ee2439928cf0 is now active with session: 2b65f031-b6f3-48fc-9839-18f70e214890","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886628354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886640194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478,"session_id":"2b65f031-b6f3-48fc-9839-18f70e214890"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:22.886652535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 236 len 32768 data: 6 7 7 7 5 5 5 4 02/30 Write block 360 len 32768 data: 5 6 6 8 7 7 8 7 03/30 Read block 115 len 12288 04/30 Read block 489 len 20480 05/30 Write block 31 len 8192 data: 6 6 06/30 Write block 360 len 40960 data: 6 7 7 9 8 8 9 8 9 8 07/30 Write block 264 len 36864 data: 4 4 5 6 3 5 5 5 6 08/30 Write block 425 len 16384 data: 8 8 7 8 09/30 Read block 248 len 28672 10/30 Read block 567 len 4096 11/30 Write block 241 len 24576 data: 6 6 5 4 3 2 12/30 Write block 407 len 12288 data: 9 9 8 13/30 Read block 33 len 16384 14/30 Flush 15/30 Write block 64 len 16384 data: 10 11 12 12 16/30 Write block 439 len 16384 data: 2 2 3 3 17/30 Read block 141 len 20480 18/30 Read block 212 len 4096 19/30 Write block 184 len 16384 data: 3 4 3 3 20/30 Read block 155 len 36864 21/30 Read block 437 len 4096 22/30 Read block 189 len 8192 23/30 Write block 162 len 20480 data: 12 8 8 8 7 24/30 Read block 198 len 4096 25/30 Read block 552 len 20480 26/30 Write block 463 len 32768 data: 7 6 5 5 5 5 5 5 27/30 Write block 286 len 32768 data: 10 10 8 8 7 8 8 7 28/30 Read block 157 len 12288 29/30 Write block 569 len 40960 data: 7 8 7 6 7 8 7 7 6 7 30/30 Read block 514 len 40960 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 4 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 14 15 16 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:22.906068349Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 74 74 74 127 127 127 F F F 2 040-059 73 73 73 126 126 126 F F F 3 060-079 73 73 73 126 126 126 T T T 4 080-099 71 71 71 122 122 122 F F F 5 100-119 70 70 70 120 120 120 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 70 70 70 121 121 121 T T T 9 180-199 72 72 72 124 124 124 T T T 10 200-219 68 68 68 118 118 118 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 74 74 74 127 127 127 F F F 14 280-299 68 68 68 118 118 118 T T T 15 300-319 73 73 73 126 126 126 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 74 74 74 127 127 127 T T T 22 440-459 67 67 67 117 117 117 T T T 23 460-479 73 73 73 126 126 126 T T T 24 480-499 68 68 68 118 118 118 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 125 125 125 F F F 28 560-579 71 71 71 122 122 122 T T T 29 580-599 73 73 73 126 126 126 F F F Max gen: 74, Max flush: 127 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 F F F 1 020-039 74 74 74 127 127 127 F F F 2 040-059 73 73 73 126 126 126 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 71 71 71 122 122 122 F F F 5 100-119 70 70 70 120 120 120 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 75 75 75 128 128 128 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 74 74 74 127 127 127 F F F 13 260-279 74 74 74 127 127 127 F F F 14 280-299 75 75 75 128 128 128 F F F 15 300-319 73 73 73 126 126 126 F F F 16 320-339 66 66 66 112 112 112 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 75 75 75 128 128 128 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 75 75 75 128 128 128 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 68 68 68 118 118 118 F F F 27 540-559 72 72 72 125 125 125 F F F 28 560-579 75 75 75 128 128 128 F F F 29 580-599 73 73 73 126 126 126 F F F Max gen: 75, Max flush: 128 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.183593989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:27.225930571Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.22630682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.25766501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.258218093Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.258256204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.258271184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.258286634Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486} {"msg":"Crucible 1a42108a-8478-4d33-8a0a-19b4b29d3a3b has session id: 0fd49bd4-25b1-4d13-9dee-7fd128bd209e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.258756626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Upstairs opts: Upstairs UUID: 1a42108a-8478-4d33-8a0a-19b4b29d3a3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.258803107Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Crucible stats registered with UUID: 1a42108a-8478-4d33-8a0a-19b4b29d3a3b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.258825327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288473336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288504637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486} {"msg":"1a42108a-8478-4d33-8a0a-19b4b29d3a3b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288531467Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288547828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288576389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288603329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.28863766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288673211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288709642Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288745902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288784894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288822264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50752)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.288977648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64779)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.289032209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51024)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.28906136Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.289673265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.289720656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.289753597Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.289783257Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.289859809Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.28991096Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.290401352Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.290430343Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.290445973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.290983606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291021387Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291038577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291051928Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291065648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291079158Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291095029Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291108909Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[0]R flush_numbers[0..12]: [122, 127, 126, 128, 122, 120, 122, 109, 128, 128, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291123139Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[0]R generation[0..12]: [71, 74, 73, 75, 71, 70, 71, 65, 75, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.29113738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.29115182Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[1]R flush_numbers[0..12]: [122, 127, 126, 128, 122, 120, 122, 109, 128, 128, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.2911659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[1]R generation[0..12]: [71, 74, 73, 75, 71, 70, 71, 65, 75, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291180821Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291195261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[2]R flush_numbers[0..12]: [122, 127, 126, 128, 122, 120, 122, 109, 128, 128, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291210212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[2]R generation[0..12]: [71, 74, 73, 75, 71, 70, 71, 65, 75, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291224042Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291238062Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291260453Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291273863Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291286983Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291302594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"":"downstairs","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291316464Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291328844Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"0","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291342244Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"1","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291359315Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"client":"2","":"downstairs","session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291373235Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291385886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"1a42108a-8478-4d33-8a0a-19b4b29d3a3b is now active with session: 0fd49bd4-25b1-4d13-9dee-7fd128bd209e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291399016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291411256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486,"session_id":"0fd49bd4-25b1-4d13-9dee-7fd128bd209e"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:27.291423806Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 40960 data: 6 6 7 6 6 7 8 8 7 9 02/30 Flush 03/30 Write block 396 len 4096 data: 8 04/30 Write block 583 len 12288 data: 9 8 7 05/30 Read block 92 len 28672 06/30 Read block 254 len 12288 07/30 Write block 91 len 12288 data: 5 7 7 08/30 Read block 308 len 12288 09/30 Write block 245 len 40960 data: 4 3 2 2 4 4 4 6 5 5 10/30 Read block 557 len 4096 11/30 Read block 441 len 36864 12/30 Read block 151 len 16384 13/30 Read block 217 len 24576 14/30 Read block 376 len 16384 15/30 Read block 592 len 24576 16/30 Write block 515 len 32768 data: 8 7 7 5 3 5 4 5 17/30 Read block 338 len 16384 18/30 Write block 55 len 8192 data: 11 11 19/30 Write block 432 len 24576 data: 5 4 4 4 4 3 20/30 Write block 387 len 28672 data: 5 3 4 4 4 5 5 21/30 Flush 22/30 Flush 23/30 Write block 291 len 36864 data: 9 9 8 7 7 8 11 8 5 24/30 Write block 571 len 12288 data: 8 7 8 25/30 Write block 326 len 16384 data: 6 7 6 6 26/30 Write block 384 len 28672 data: 6 5 5 6 4 5 5 27/30 Write block 10 len 4096 data: 5 28/30 Write block 539 len 28672 data: 6 7 6 6 8 8 6 29/30 Read block 249 len 32768 30/30 Read block 39 len 12288 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:27.310872091Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 122 122 122 T T T 1 020-039 74 74 74 127 127 127 F F F 2 040-059 76 76 76 130 130 130 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 70 70 70 120 120 120 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 75 75 75 128 128 128 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 76 76 76 130 130 130 F F F 13 260-279 74 74 74 127 127 127 F F F 14 280-299 75 75 75 128 128 128 T T T 15 300-319 73 73 73 126 126 126 F F F 16 320-339 66 66 66 112 112 112 T T T 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 76 76 76 130 130 130 T T T 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 75 75 75 128 128 128 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 76 76 76 130 130 130 F F F 26 520-539 76 76 76 130 130 130 T T T 27 540-559 72 72 72 125 125 125 T T T 28 560-579 75 75 75 128 128 128 T T T 29 580-599 76 76 76 130 130 130 F F F Max gen: 76, Max flush: 130 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 131 131 131 F F F 1 020-039 74 74 74 127 127 127 F F F 2 040-059 76 76 76 130 130 130 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 70 70 70 120 120 120 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 75 75 75 128 128 128 F F F 10 200-219 68 68 68 118 118 118 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 76 76 76 130 130 130 F F F 13 260-279 74 74 74 127 127 127 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 73 73 73 126 126 126 F F F 16 320-339 77 77 77 131 131 131 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 75 75 75 128 128 128 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 76 76 76 130 130 130 F F F 26 520-539 77 77 77 131 131 131 F F F 27 540-559 77 77 77 131 131 131 F F F 28 560-579 77 77 77 131 131 131 F F F 29 580-599 76 76 76 130 130 130 F F F Max gen: 77, Max flush: 131 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.614908995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.657157305Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.657792791Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.657829801Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.657844242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.657858532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494} {"msg":"Crucible 54c74343-e71d-4044-a2e7-ca4b0f9deaa1 has session id: 7b666a71-073d-4e7d-be94-0f813129283a","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.658280112Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Upstairs opts: Upstairs UUID: 54c74343-e71d-4044-a2e7-ca4b0f9deaa1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.658326453Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Crucible stats registered with UUID: 54c74343-e71d-4044-a2e7-ca4b0f9deaa1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.658352374Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688100825Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688137146Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494} {"msg":"54c74343-e71d-4044-a2e7-ca4b0f9deaa1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688188827Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688214828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688252759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688273299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.6882908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.68830774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.68832512Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688341781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688359971Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688377231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39402)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688443333Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53136)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688490624Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49826)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.688529495Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689195951Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689225882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689349805Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689429247Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689457337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689472708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689750574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.689849397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690035411Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690284357Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690319598Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690336208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.69040609Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.6904255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690439471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690592624Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690611155Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[0]R flush_numbers[0..12]: [131, 127, 130, 128, 130, 120, 122, 109, 128, 128, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690629065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[0]R generation[0..12]: [77, 74, 76, 75, 76, 70, 71, 65, 75, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690645846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690660456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[1]R flush_numbers[0..12]: [131, 127, 130, 128, 130, 120, 122, 109, 128, 128, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690674326Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[1]R generation[0..12]: [77, 74, 76, 75, 76, 70, 71, 65, 75, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690688117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690702177Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[2]R flush_numbers[0..12]: [131, 127, 130, 128, 130, 120, 122, 109, 128, 128, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690716047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[2]R generation[0..12]: [77, 74, 76, 75, 76, 70, 71, 65, 75, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690729728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690743798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690756979Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690770209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690783219Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2024-12-09T15:10:31.690796219Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"":"downstairs","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.69081906Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.69083123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"0","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.69084515Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"1","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690860311Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"client":"2","":"downstairs","session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690874111Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690886611Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"54c74343-e71d-4044-a2e7-ca4b0f9deaa1 is now active with session: 7b666a71-073d-4e7d-be94-0f813129283a","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690899662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690911642Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494,"session_id":"7b666a71-073d-4e7d-be94-0f813129283a"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:31.690923972Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 40960 data: 9 11 13 12 10 9 9 8 7 6 02/30 Read block 33 len 20480 03/30 Write block 527 len 32768 data: 4 4 4 5 6 8 6 6 04/30 Read block 518 len 4096 05/30 Read block 570 len 20480 06/30 Read block 540 len 8192 07/30 Read block 84 len 32768 08/30 Write block 465 len 24576 data: 6 6 6 6 6 6 09/30 Read block 139 len 40960 10/30 Read block 27 len 36864 11/30 Write block 56 len 16384 data: 12 9 9 9 12/30 Write block 266 len 32768 data: 6 7 4 6 6 6 7 8 13/30 Write block 204 len 16384 data: 5 5 5 6 14/30 Flush 15/30 Write block 300 len 28672 data: 4 5 5 5 5 6 8 16/30 Write block 306 len 24576 data: 9 8 7 5 6 6 17/30 Read block 149 len 12288 18/30 Read block 39 len 12288 19/30 Read block 15 len 24576 20/30 Write block 192 len 32768 data: 5 7 8 7 7 9 7 7 21/30 Write block 515 len 32768 data: 9 8 8 6 4 6 5 6 22/30 Write block 542 len 28672 data: 7 9 9 7 5 5 5 23/30 Write block 11 len 12288 data: 4 4 5 24/30 Write block 430 len 16384 data: 7 6 6 5 25/30 Read block 384 len 8192 26/30 Write block 449 len 24576 data: 3 3 4 4 5 6 27/30 Read block 576 len 40960 28/30 Write block 41 len 8192 data: 8 8 29/30 Write block 523 len 16384 data: 5 4 4 3 30/30 Read block 266 len 16384 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 16 14 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:31.710543261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 131 131 131 T T T 1 020-039 74 74 74 127 127 127 F F F 2 040-059 78 78 78 132 132 132 T T T 3 060-079 75 75 75 128 128 128 F F F 4 080-099 78 78 78 132 132 132 F F F 5 100-119 78 78 78 132 132 132 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 75 75 75 128 128 128 T T T 10 200-219 78 78 78 132 132 132 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 76 76 76 130 130 130 F F F 13 260-279 78 78 78 132 132 132 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 73 73 73 126 126 126 T T T 16 320-339 77 77 77 131 131 131 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 76 76 76 130 130 130 T T T 22 440-459 75 75 75 128 128 128 T T T 23 460-479 78 78 78 132 132 132 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 76 76 76 130 130 130 T T T 26 520-539 78 78 78 132 132 132 T T T 27 540-559 77 77 77 131 131 131 T T T 28 560-579 77 77 77 131 131 131 F F F 29 580-599 76 76 76 130 130 130 F F F Max gen: 78, Max flush: 132 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 133 133 133 F F F 1 020-039 74 74 74 127 127 127 F F F 2 040-059 79 79 79 133 133 133 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 78 78 78 132 132 132 F F F 5 100-119 78 78 78 132 132 132 F F F 6 120-139 71 71 71 122 122 122 F F F 7 140-159 65 65 65 109 109 109 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 79 79 79 133 133 133 F F F 10 200-219 78 78 78 132 132 132 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 76 76 76 130 130 130 F F F 13 260-279 78 78 78 132 132 132 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 79 79 79 133 133 133 F F F 16 320-339 77 77 77 131 131 131 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 74 74 74 127 127 127 F F F 21 420-439 79 79 79 133 133 133 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 78 78 78 132 132 132 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 79 79 79 133 133 133 F F F 26 520-539 79 79 79 133 133 133 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 77 77 77 131 131 131 F F F 29 580-599 76 76 76 130 130 130 F F F Max gen: 79, Max flush: 133 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:35.997968447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:36.040200067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.040596997Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.071416773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.071993527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.072033368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.072063729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.072079989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502} {"msg":"Crucible 8c0c50d2-c03c-4ca8-ac17-b35bbd016a69 has session id: e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.072512349Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Upstairs opts: Upstairs UUID: 8c0c50d2-c03c-4ca8-ac17-b35bbd016a69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.07256269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Crucible stats registered with UUID: 8c0c50d2-c03c-4ca8-ac17-b35bbd016a69","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.072589841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102520047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102564098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502} {"msg":"8c0c50d2-c03c-4ca8-ac17-b35bbd016a69 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102632819Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.10266131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102687931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102703021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102717241Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102730922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102744542Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102757972Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102772183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102786313Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_connection connected from Ok(127.0.0.1:35647)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102900346Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_connection connected from Ok(127.0.0.1:47788)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102938997Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_connection connected from Ok(127.0.0.1:45277)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.102969728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.103590372Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.103632383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.103665694Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.103701765Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.103731656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.103760826Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104275229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.10431996Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104504044Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104895354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104925264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104940775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104953755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104967455Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.104980916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105022277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105039597Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[0]R flush_numbers[0..12]: [133, 127, 133, 128, 132, 132, 122, 109, 128, 133, 132, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105063438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[0]R generation[0..12]: [79, 74, 79, 75, 78, 78, 71, 65, 75, 79, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105081908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105096888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[1]R flush_numbers[0..12]: [133, 127, 133, 128, 132, 132, 122, 109, 128, 133, 132, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105110839Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[1]R generation[0..12]: [79, 74, 79, 75, 78, 78, 71, 65, 75, 79, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105124549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105138749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[2]R flush_numbers[0..12]: [133, 127, 133, 128, 132, 132, 122, 109, 128, 133, 132, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.10515261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[2]R generation[0..12]: [79, 74, 79, 75, 78, 78, 71, 65, 75, 79, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.10516636Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.10518039Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105193681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105207021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105220121Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:10:36.105233122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"":"downstairs","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105252872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105265192Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"0","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105280253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"1","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105293873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"client":"2","":"downstairs","session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105307383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105319813Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"8c0c50d2-c03c-4ca8-ac17-b35bbd016a69 is now active with session: e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105332804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105344814Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502,"session_id":"e6a5c375-ab67-47c3-a0a8-3ce5fb1cea30"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:36.105357274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 216 len 4096 data: 6 02/30 Write block 572 len 36864 data: 8 9 9 8 8 7 8 8 8 03/30 Read block 382 len 12288 04/30 Flush 05/30 Flush 06/30 Write block 382 len 4096 data: 7 07/30 Write block 253 len 36864 data: 6 6 7 7 6 8 7 6 5 08/30 Flush 09/30 Write block 567 len 16384 data: 5 5 8 9 10/30 Write block 470 len 16384 data: 7 6 4 4 11/30 Read block 191 len 36864 12/30 Write block 50 len 4096 data: 8 13/30 Write block 323 len 20480 data: 5 5 5 7 8 14/30 Read block 30 len 24576 15/30 Write block 187 len 36864 data: 4 2 2 2 4 6 8 9 8 16/30 Read block 392 len 4096 17/30 Write block 410 len 16384 data: 8 8 9 8 18/30 Write block 503 len 8192 data: 8 7 19/30 Write block 412 len 20480 data: 10 9 8 6 5 20/30 Write block 120 len 28672 data: 5 5 5 5 4 4 5 21/30 Write block 333 len 24576 data: 6 7 7 6 6 7 22/30 Read block 149 len 40960 23/30 Read block 292 len 32768 24/30 Read block 374 len 4096 25/30 Flush 26/30 Write block 127 len 16384 data: 7 5 5 5 27/30 Write block 148 len 40960 data: 6 5 5 3 5 9 11 12 13 12 28/30 Write block 319 len 36864 data: 7 5 6 5 6 6 6 8 9 29/30 Write block 42 len 32768 data: 9 10 8 8 9 7 5 7 30/30 Read block 382 len 12288 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:36.124283337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 133 133 133 F F F 1 020-039 74 74 74 127 127 127 F F F 2 040-059 80 80 80 137 137 137 T T T 3 060-079 75 75 75 128 128 128 F F F 4 080-099 78 78 78 132 132 132 F F F 5 100-119 78 78 78 132 132 132 F F F 6 120-139 80 80 80 137 137 137 T T T 7 140-159 65 65 65 109 109 109 T T T 8 160-179 75 75 75 128 128 128 F F F 9 180-199 80 80 80 137 137 137 F F F 10 200-219 80 80 80 134 134 134 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 80 80 80 136 136 136 F F F 13 260-279 80 80 80 136 136 136 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 79 79 79 133 133 133 T T T 16 320-339 80 80 80 137 137 137 T T T 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 80 80 80 136 136 136 F F F 20 400-419 80 80 80 137 137 137 F F F 21 420-439 79 79 79 133 133 133 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 80 80 80 137 137 137 F F F 26 520-539 79 79 79 133 133 133 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 80, Max flush: 137 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 133 133 133 F F F 1 020-039 74 74 74 127 127 127 F F F 2 040-059 81 81 81 138 138 138 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 78 78 78 132 132 132 F F F 5 100-119 78 78 78 132 132 132 F F F 6 120-139 81 81 81 138 138 138 F F F 7 140-159 81 81 81 138 138 138 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 80 80 80 137 137 137 F F F 10 200-219 80 80 80 134 134 134 F F F 11 220-239 74 74 74 127 127 127 F F F 12 240-259 80 80 80 136 136 136 F F F 13 260-279 80 80 80 136 136 136 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 74 74 74 127 127 127 F F F 19 380-399 80 80 80 136 136 136 F F F 20 400-419 80 80 80 137 137 137 F F F 21 420-439 79 79 79 133 133 133 F F F 22 440-459 79 79 79 133 133 133 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 80 80 80 137 137 137 F F F 26 520-539 79 79 79 133 133 133 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 81, Max flush: 138 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.412852891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.456663428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.457318434Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.457354885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.457369915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.457384445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510} {"msg":"Crucible 88895a6f-944c-4fd5-a638-e4c6c83bec47 has session id: ecb2ee10-9eac-44dd-a638-16cb87fb9212","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.457795005Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Upstairs opts: Upstairs UUID: 88895a6f-944c-4fd5-a638-e4c6c83bec47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.457831916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Crucible stats registered with UUID: 88895a6f-944c-4fd5-a638-e4c6c83bec47","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.457849397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.48726927Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487296031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510} {"msg":"88895a6f-944c-4fd5-a638-e4c6c83bec47 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487354112Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487372693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487397453Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487412573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487439164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487468405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487539136Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487568347Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487590578Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487614748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_connection connected from Ok(127.0.0.1:52702)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487642669Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_connection connected from Ok(127.0.0.1:63523)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.48767312Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_connection connected from Ok(127.0.0.1:39073)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.487857674Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.488342166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.488369946Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.488495419Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.48853172Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.488626582Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.488665263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489114324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489154565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489294528Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489676997Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489713428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489731249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489744099Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.48975773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.48977124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.48978343Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.48979662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[0]R flush_numbers[0..12]: [133, 127, 138, 128, 132, 132, 138, 138, 128, 137, 134, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489811001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[0]R generation[0..12]: [79, 74, 81, 75, 78, 78, 81, 81, 75, 80, 80, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489824981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489839231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[1]R flush_numbers[0..12]: [133, 127, 138, 128, 132, 132, 138, 138, 128, 137, 134, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489853342Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[1]R generation[0..12]: [79, 74, 81, 75, 78, 78, 81, 81, 75, 80, 80, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489870882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489885653Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[2]R flush_numbers[0..12]: [133, 127, 138, 128, 132, 132, 138, 138, 128, 137, 134, 127]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489899863Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[2]R generation[0..12]: [79, 74, 81, 75, 78, 78, 81, 81, 75, 80, 80, 74]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489913613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489927703Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489948914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489962294Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489975295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.489988365Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"":"downstairs","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490001875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490014325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"0","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490027846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"1","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490041306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"client":"2","":"downstairs","session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490054657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490066877Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"88895a6f-944c-4fd5-a638-e4c6c83bec47 is now active with session: ecb2ee10-9eac-44dd-a638-16cb87fb9212","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490081677Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490094258Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510,"session_id":"ecb2ee10-9eac-44dd-a638-16cb87fb9212"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:40.490106628Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 394 len 40960 data: 6 7 9 9 9 7 6 8 10 12 02/30 Write block 250 len 32768 data: 5 5 7 7 7 8 8 7 03/30 Write block 73 len 32768 data: 7 6 5 4 7 7 6 5 04/30 Read block 186 len 4096 05/30 Read block 327 len 4096 06/30 Flush 07/30 Write block 222 len 8192 data: 9 9 08/30 Write block 517 len 28672 data: 9 7 5 7 6 7 6 09/30 Write block 439 len 8192 data: 3 3 10/30 Read block 315 len 36864 11/30 Read block 46 len 4096 12/30 Read block 59 len 8192 13/30 Read block 180 len 24576 14/30 Write block 267 len 16384 data: 8 5 7 7 15/30 Read block 196 len 24576 16/30 Write block 347 len 12288 data: 12 8 8 17/30 Read block 22 len 12288 18/30 Read block 344 len 20480 19/30 Read block 507 len 20480 20/30 Write block 81 len 20480 data: 6 6 5 5 4 21/30 Write block 9 len 8192 data: 6 6 22/30 Write block 367 len 12288 data: 9 10 9 23/30 Read block 285 len 8192 24/30 Write block 63 len 24576 data: 9 11 12 13 13 9 25/30 Read block 101 len 4096 26/30 Flush 27/30 Write block 31 len 12288 data: 7 7 6 28/30 Write block 257 len 16384 data: 8 9 8 7 29/30 Write block 119 len 32768 data: 7 6 6 6 6 5 5 6 30/30 Read block 477 len 32768 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:40.508373424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 140 140 140 F F F 1 020-039 74 74 74 127 127 127 T T T 2 040-059 81 81 81 138 138 138 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 78 78 78 132 132 132 T T T 6 120-139 81 81 81 138 138 138 T T T 7 140-159 81 81 81 138 138 138 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 80 80 80 137 137 137 F F F 10 200-219 80 80 80 134 134 134 F F F 11 220-239 82 82 82 140 140 140 F F F 12 240-259 82 82 82 139 139 139 T T T 13 260-279 82 82 82 140 140 140 T T T 14 280-299 77 77 77 131 131 131 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 F F F 17 340-359 82 82 82 140 140 140 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 82 82 82 139 139 139 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 140 140 140 F F F 22 440-459 82 82 82 140 140 140 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 82 82 82 140 140 140 F F F 26 520-539 82 82 82 140 140 140 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 82, Max flush: 140 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 140 140 140 F F F 1 020-039 83 83 83 141 141 141 F F F 2 040-059 81 81 81 138 138 138 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 83 83 83 141 141 141 F F F 6 120-139 83 83 83 141 141 141 F F F 7 140-159 81 81 81 138 138 138 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 80 80 80 137 137 137 F F F 10 200-219 80 80 80 134 134 134 F F F 11 220-239 82 82 82 140 140 140 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 83 83 83 141 141 141 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 F F F 17 340-359 82 82 82 140 140 140 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 82 82 82 139 139 139 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 140 140 140 F F F 22 440-459 82 82 82 140 140 140 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 68 68 68 118 118 118 F F F 25 500-519 82 82 82 140 140 140 F F F 26 520-539 82 82 82 140 140 140 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 83, Max flush: 141 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.766128562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:44.807406309Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.807796108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.838465011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.839008684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.839041285Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.839055725Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.839069856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518} {"msg":"Crucible 09d6f2ee-f2fa-42d5-a052-a1ef8faf665f has session id: b387331f-04c6-4f30-9bdd-b2736dcc75fe","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.839489386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Upstairs opts: Upstairs UUID: 09d6f2ee-f2fa-42d5-a052-a1ef8faf665f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.839530277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Crucible stats registered with UUID: 09d6f2ee-f2fa-42d5-a052-a1ef8faf665f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.839548037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.86937705Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869420641Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518} {"msg":"09d6f2ee-f2fa-42d5-a052-a1ef8faf665f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869486183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869514984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869540864Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869555785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869569735Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869583535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869605786Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869633756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869659037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869674317Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:39539)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869921773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:49492)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869967524Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:46451)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.869989255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.870555918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.87059775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.870667781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.870711382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.870780764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.870809644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871170673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871267106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871347127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871804138Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871845879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.87187613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.87190149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871930651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871956992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.871978083Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872002763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[0]R flush_numbers[0..12]: [140, 141, 138, 140, 140, 141, 141, 138, 128, 137, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872027834Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[0]R generation[0..12]: [82, 83, 81, 82, 82, 83, 83, 81, 75, 80, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872053404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872078415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[1]R flush_numbers[0..12]: [140, 141, 138, 140, 140, 141, 141, 138, 128, 137, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872103746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[1]R generation[0..12]: [82, 83, 81, 82, 82, 83, 83, 81, 75, 80, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872130546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872158587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[2]R flush_numbers[0..12]: [140, 141, 138, 140, 140, 141, 141, 138, 128, 137, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872189997Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[2]R generation[0..12]: [82, 83, 81, 82, 82, 83, 83, 81, 75, 80, 80, 82]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:10:44.872220408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872251929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.87228074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.87230921Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872337181Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872366462Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"":"downstairs","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872395502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872425753Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"0","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872454894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"1","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872475224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"client":"2","":"downstairs","session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872489645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872502415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"09d6f2ee-f2fa-42d5-a052-a1ef8faf665f is now active with session: b387331f-04c6-4f30-9bdd-b2736dcc75fe","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872515405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872527545Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518,"session_id":"b387331f-04c6-4f30-9bdd-b2736dcc75fe"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:44.872542616Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 99 len 28672 data: 12 14 13 11 10 10 9 03/30 Read block 417 len 16384 04/30 Read block 328 len 4096 05/30 Write block 287 len 24576 data: 11 9 9 8 10 10 06/30 Write block 449 len 20480 data: 4 4 5 5 6 07/30 Read block 296 len 12288 08/30 Read block 375 len 16384 09/30 Write block 140 len 32768 data: 5 8 7 7 7 6 7 6 10/30 Write block 139 len 16384 data: 4 6 9 8 11/30 Read block 378 len 24576 12/30 Flush 13/30 Write block 83 len 12288 data: 6 6 5 14/30 Read block 26 len 40960 15/30 Read block 277 len 12288 16/30 Flush 17/30 Write block 7 len 16384 data: 6 5 7 7 18/30 Read block 156 len 8192 19/30 Write block 487 len 12288 data: 8 6 6 20/30 Write block 10 len 24576 data: 8 5 5 6 7 8 21/30 Read block 532 len 20480 22/30 Flush 23/30 Write block 120 len 36864 data: 7 7 7 7 6 6 7 8 6 24/30 Read block 411 len 4096 25/30 Write block 119 len 8192 data: 8 8 26/30 Write block 180 len 32768 data: 8 6 5 5 4 5 4 5 27/30 Read block 417 len 12288 28/30 Read block 341 len 20480 29/30 Write block 119 len 16384 data: 9 9 8 8 30/30 Write block 345 len 36864 data: 10 12 13 9 9 7 6 7 5 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:44.891447568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 145 145 145 F F F 1 020-039 83 83 83 141 141 141 F F F 2 040-059 81 81 81 138 138 138 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 84 84 84 144 144 144 F F F 5 100-119 84 84 84 143 143 143 T T T 6 120-139 84 84 84 143 143 143 T T T 7 140-159 84 84 84 143 143 143 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 80 80 80 137 137 137 T T T 10 200-219 80 80 80 134 134 134 F F F 11 220-239 82 82 82 140 140 140 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 83 83 83 141 141 141 F F F 14 280-299 84 84 84 143 143 143 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 F F F 17 340-359 82 82 82 140 140 140 T T T 18 360-379 82 82 82 140 140 140 F F F 19 380-399 82 82 82 139 139 139 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 140 140 140 F F F 22 440-459 84 84 84 143 143 143 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 84 84 84 145 145 145 F F F 25 500-519 82 82 82 140 140 140 F F F 26 520-539 82 82 82 140 140 140 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 84, Max flush: 145 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 145 145 145 F F F 1 020-039 83 83 83 141 141 141 F F F 2 040-059 81 81 81 138 138 138 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 84 84 84 144 144 144 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 85 85 85 146 146 146 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 80 80 80 134 134 134 F F F 11 220-239 82 82 82 140 140 140 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 83 83 83 141 141 141 F F F 14 280-299 84 84 84 143 143 143 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 F F F 17 340-359 85 85 85 146 146 146 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 82 82 82 139 139 139 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 140 140 140 F F F 22 440-459 84 84 84 143 143 143 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 84 84 84 145 145 145 F F F 25 500-519 82 82 82 140 140 140 F F F 26 520-539 82 82 82 140 140 140 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 85, Max flush: 146 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.178594388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:49.220201023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.220580322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.2514477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.252063635Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.252100746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.252119566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.252134916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526} {"msg":"Crucible d6dcdf3f-bbfb-4091-8bec-c6752f079eee has session id: 15ed60f2-bcee-42a4-82c4-9082a7513a6d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.252550926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Upstairs opts: Upstairs UUID: d6dcdf3f-bbfb-4091-8bec-c6752f079eee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.252593147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Crucible stats registered with UUID: d6dcdf3f-bbfb-4091-8bec-c6752f079eee","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.252611258Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282440891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282472181Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526} {"msg":"d6dcdf3f-bbfb-4091-8bec-c6752f079eee active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282543493Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282565304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282595155Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282625395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282660026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282691427Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282718767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282749728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282786559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.282807309Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52148)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.28282501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35079)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.28284259Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58924)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.283016365Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.283569028Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.283612299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.28364728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.28367884Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.283708191Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.283739222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.284283695Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.284314096Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.284337386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.284867719Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.284896819Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.28491203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.284982072Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285003782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285017782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285030173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285043693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[0]R flush_numbers[0..12]: [145, 141, 138, 140, 144, 146, 146, 143, 128, 146, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285057953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[0]R generation[0..12]: [84, 83, 81, 82, 84, 85, 85, 84, 75, 85, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285071894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285086164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[1]R flush_numbers[0..12]: [145, 141, 138, 140, 144, 146, 146, 143, 128, 146, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285100254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[1]R generation[0..12]: [84, 83, 81, 82, 84, 85, 85, 84, 75, 85, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285113965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285127835Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[2]R flush_numbers[0..12]: [145, 141, 138, 140, 144, 146, 146, 143, 128, 146, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285141625Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[2]R generation[0..12]: [84, 83, 81, 82, 84, 85, 85, 84, 75, 85, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285158466Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285172886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285186206Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285199577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285212477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"All extents match"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285264198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"":"downstairs","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285280869Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285294209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"0","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285307639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"1","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.28532091Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"client":"2","":"downstairs","session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.2853344Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.2853469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"d6dcdf3f-bbfb-4091-8bec-c6752f079eee is now active with session: 15ed60f2-bcee-42a4-82c4-9082a7513a6d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.28535995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285372171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526,"session_id":"15ed60f2-bcee-42a4-82c4-9082a7513a6d"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:49.285384511Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 38 len 4096 data: 8 02/30 Read block 504 len 36864 03/30 Write block 267 len 28672 data: 9 6 8 8 7 8 9 04/30 Write block 481 len 36864 data: 14 11 9 10 9 7 9 7 7 05/30 Read block 76 len 4096 06/30 Flush 07/30 Write block 216 len 40960 data: 7 6 5 6 7 8 10 10 10 9 08/30 Write block 161 len 28672 data: 12 13 9 9 9 8 5 09/30 Read block 138 len 16384 10/30 Read block 12 len 12288 11/30 Write block 386 len 24576 data: 6 7 5 6 6 5 12/30 Write block 562 len 8192 data: 4 4 13/30 Read block 91 len 4096 14/30 Write block 34 len 12288 data: 6 5 6 15/30 Read block 406 len 12288 16/30 Flush 17/30 Flush 18/30 Write block 260 len 16384 data: 8 6 5 4 19/30 Flush 20/30 Write block 235 len 8192 data: 7 7 21/30 Read block 510 len 16384 22/30 Flush 23/30 Write block 346 len 40960 data: 13 14 10 10 8 7 8 6 5 4 24/30 Flush 25/30 Write block 490 len 24576 data: 7 7 5 5 7 6 26/30 Read block 416 len 12288 27/30 Read block 400 len 8192 28/30 Read block 261 len 24576 29/30 Read block 568 len 24576 30/30 Read block 61 len 36864 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:49.304911758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 145 145 145 F F F 1 020-039 86 86 86 148 148 148 F F F 2 040-059 81 81 81 138 138 138 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 84 84 84 144 144 144 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 85 85 85 146 146 146 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 86 86 86 148 148 148 F F F 11 220-239 86 86 86 151 151 151 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 84 84 84 143 143 143 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 F F F 17 340-359 86 86 86 152 152 152 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 140 140 140 F F F 22 440-459 84 84 84 143 143 143 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 86 86 86 147 147 147 T T T 25 500-519 82 82 82 140 140 140 F F F 26 520-539 82 82 82 140 140 140 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 86 86 86 148 148 148 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 86, Max flush: 152 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 145 145 145 F F F 1 020-039 86 86 86 148 148 148 F F F 2 040-059 81 81 81 138 138 138 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 84 84 84 144 144 144 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 85 85 85 146 146 146 F F F 7 140-159 84 84 84 143 143 143 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 86 86 86 148 148 148 F F F 11 220-239 86 86 86 151 151 151 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 84 84 84 143 143 143 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 F F F 17 340-359 86 86 86 152 152 152 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 140 140 140 F F F 22 440-459 84 84 84 143 143 143 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 87 87 87 153 153 153 F F F 25 500-519 82 82 82 140 140 140 F F F 26 520-539 82 82 82 140 140 140 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 86 86 86 148 148 148 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 87, Max flush: 153 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.574399086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.616965033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.617601239Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.617636519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.61765085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.61766532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534} {"msg":"Crucible 82307fc1-6bc0-4565-8319-089b2d11e74f has session id: abbfc13e-80c5-40b7-ad03-f3a4e03ca394","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.618131681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Upstairs opts: Upstairs UUID: 82307fc1-6bc0-4565-8319-089b2d11e74f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.618181712Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Crucible stats registered with UUID: 82307fc1-6bc0-4565-8319-089b2d11e74f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.618207293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.647882882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.647917923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534} {"msg":"82307fc1-6bc0-4565-8319-089b2d11e74f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.647938414Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.647954684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.647985385Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648005465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648023226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648040156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648056956Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648073487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648098237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648132188Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_connection connected from Ok(127.0.0.1:46575)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648174489Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_connection connected from Ok(127.0.0.1:33172)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648392855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_connection connected from Ok(127.0.0.1:35181)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648445476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.648978758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.649007659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.64902292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.64903681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.6490506Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.649064221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.649553292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.649697816Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.649825739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650117546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650140846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650154957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650268219Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.65028965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.65030683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650345051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650361802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[0]R flush_numbers[0..12]: [145, 148, 138, 140, 144, 146, 146, 143, 148, 146, 148, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650379732Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[0]R generation[0..12]: [84, 86, 81, 82, 84, 85, 85, 84, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650393772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650408143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[1]R flush_numbers[0..12]: [145, 148, 138, 140, 144, 146, 146, 143, 148, 146, 148, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650422233Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[1]R generation[0..12]: [84, 86, 81, 82, 84, 85, 85, 84, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650436023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650450204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[2]R flush_numbers[0..12]: [145, 148, 138, 140, 144, 146, 146, 143, 148, 146, 148, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650464154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[2]R generation[0..12]: [84, 86, 81, 82, 84, 85, 85, 84, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650477784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650491905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650505075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650518225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650534376Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"All extents match","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-12-09T15:10:53.650556226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"":"downstairs","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650569957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650582217Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"0","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650595517Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"1","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650608757Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"client":"2","":"downstairs","session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650622058Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650634698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"82307fc1-6bc0-4565-8319-089b2d11e74f is now active with session: abbfc13e-80c5-40b7-ad03-f3a4e03ca394","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650647709Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650659609Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534,"session_id":"abbfc13e-80c5-40b7-ad03-f3a4e03ca394"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:53.650672049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 123 len 16384 data: 8 7 7 8 02/30 Write block 47 len 36864 data: 8 6 8 9 6 8 7 9 12 03/30 Write block 421 len 40960 data: 5 7 7 7 9 9 8 9 8 8 04/30 Write block 134 len 32768 data: 4 4 5 3 4 5 7 10 05/30 Flush 06/30 Write block 120 len 32768 data: 10 9 9 9 8 8 9 9 07/30 Write block 439 len 20480 data: 4 4 4 4 4 08/30 Write block 46 len 36864 data: 10 9 7 9 10 7 9 8 10 09/30 Write block 10 len 16384 data: 9 6 6 7 10/30 Write block 143 len 20480 data: 8 8 7 8 7 11/30 Write block 203 len 4096 data: 7 12/30 Read block 575 len 24576 13/30 Write block 359 len 4096 data: 5 14/30 Read block 175 len 8192 15/30 Read block 161 len 24576 16/30 Write block 31 len 28672 data: 8 8 7 7 6 7 8 17/30 Read block 562 len 8192 18/30 Flush 19/30 Write block 260 len 12288 data: 9 7 6 20/30 Read block 121 len 28672 21/30 Flush 22/30 Write block 528 len 12288 data: 5 5 6 23/30 Write block 332 len 20480 data: 5 7 8 8 7 24/30 Write block 427 len 32768 data: 9 10 9 9 7 7 6 5 25/30 Read block 439 len 24576 26/30 Read block 567 len 36864 27/30 Write block 486 len 20480 data: 8 10 8 8 8 28/30 Write block 65 len 8192 data: 13 14 29/30 Read block 476 len 4096 30/30 Write block 571 len 32768 data: 9 9 10 10 9 9 8 9 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:53.668500365Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 155 155 155 F F F 1 020-039 88 88 88 155 155 155 F F F 2 040-059 88 88 88 155 155 155 F F F 3 060-079 82 82 82 140 140 140 T T T 4 080-099 84 84 84 144 144 144 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 88 88 88 155 155 155 F F F 7 140-159 88 88 88 155 155 155 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 86 86 86 151 151 151 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 88 88 88 156 156 156 F F F 14 280-299 84 84 84 143 143 143 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 81 81 81 138 138 138 T T T 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 88 88 88 155 155 155 T T T 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 87 87 87 153 153 153 T T T 25 500-519 82 82 82 140 140 140 F F F 26 520-539 82 82 82 140 140 140 T T T 27 540-559 79 79 79 133 133 133 F F F 28 560-579 86 86 86 148 148 148 T T T 29 580-599 80 80 80 134 134 134 F F F Max gen: 88, Max flush: 156 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 155 155 155 F F F 1 020-039 88 88 88 155 155 155 F F F 2 040-059 88 88 88 155 155 155 F F F 3 060-079 89 89 89 157 157 157 F F F 4 080-099 84 84 84 144 144 144 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 88 88 88 155 155 155 F F F 7 140-159 88 88 88 155 155 155 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 86 86 86 151 151 151 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 88 88 88 156 156 156 F F F 14 280-299 84 84 84 143 143 143 F F F 15 300-319 81 81 81 138 138 138 F F F 16 320-339 89 89 89 157 157 157 F F F 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 89 89 89 157 157 157 F F F 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 89 89 89 157 157 157 F F F 25 500-519 82 82 82 140 140 140 F F F 26 520-539 89 89 89 157 157 157 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 89 89 89 157 157 157 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 89, Max flush: 157 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:57.935471513Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:57.976621566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:57.976979945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.007686599Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.008236922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.008272273Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.008287203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.008301854Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542} {"msg":"Crucible 6f495aa0-3d4e-4363-8e3c-c36fce800b8f has session id: 2e29e6ba-2c92-4d5d-9b9c-2b03c5567010","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.008660532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Upstairs opts: Upstairs UUID: 6f495aa0-3d4e-4363-8e3c-c36fce800b8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.008705263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Crucible stats registered with UUID: 6f495aa0-3d4e-4363-8e3c-c36fce800b8f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.008735264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038210348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038243879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542} {"msg":"6f495aa0-3d4e-4363-8e3c-c36fce800b8f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038315171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038338621Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038367402Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038385353Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038402803Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038421114Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038450224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038469804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038487585Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038504896Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_connection connected from Ok(127.0.0.1:53230)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038610498Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_connection connected from Ok(127.0.0.1:48535)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.038649859Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_connection connected from Ok(127.0.0.1:61095)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.03868544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.039437028Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.039468639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.039486729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.039500829Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.039614282Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.039660123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040051852Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040196916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.04035092Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040648947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040676107Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040691088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040749899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040769779Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.04078374Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.04081154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040826191Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[0]R flush_numbers[0..12]: [155, 155, 155, 157, 144, 146, 155, 155, 148, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040864912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[0]R generation[0..12]: [88, 88, 88, 89, 84, 85, 88, 88, 86, 85, 88, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040882242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040896573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[1]R flush_numbers[0..12]: [155, 155, 155, 157, 144, 146, 155, 155, 148, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040910603Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[1]R generation[0..12]: [88, 88, 88, 89, 84, 85, 88, 88, 86, 85, 88, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040924423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040938543Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[2]R flush_numbers[0..12]: [155, 155, 155, 157, 144, 146, 155, 155, 148, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040952264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[2]R generation[0..12]: [88, 88, 88, 89, 84, 85, 88, 88, 86, 85, 88, 86]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040965734Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040979644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.040992835Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041006305Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041019676Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041043836Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"":"downstairs","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041058046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041070077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"0","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041083387Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"1","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041096557Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"client":"2","":"downstairs","session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041109658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041121948Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"6f495aa0-3d4e-4363-8e3c-c36fce800b8f is now active with session: 2e29e6ba-2c92-4d5d-9b9c-2b03c5567010","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041134678Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:10:58.041146748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542,"session_id":"2e29e6ba-2c92-4d5d-9b9c-2b03c5567010"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:58.041161619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 226 len 32768 data: 7 7 5 6 6 7 9 11 02/30 Flush 03/30 Write block 437 len 12288 data: 4 3 5 04/30 Write block 116 len 20480 data: 5 5 5 10 11 05/30 Read block 54 len 12288 06/30 Write block 128 len 36864 data: 7 6 6 5 5 5 5 5 6 07/30 Read block 278 len 20480 08/30 Write block 140 len 32768 data: 8 11 9 9 9 8 9 8 09/30 Flush 10/30 Write block 506 len 16384 data: 8 9 9 8 11/30 Write block 55 len 8192 data: 13 13 12/30 Read block 70 len 36864 13/30 Write block 191 len 4096 data: 5 14/30 Read block 351 len 16384 15/30 Write block 42 len 4096 data: 10 16/30 Write block 126 len 20480 data: 10 10 8 7 7 17/30 Write block 121 len 28672 data: 10 10 10 9 9 11 11 18/30 Write block 283 len 20480 data: 6 7 8 11 12 19/30 Write block 313 len 40960 data: 6 6 7 8 8 6 8 6 7 6 20/30 Write block 323 len 32768 data: 7 7 7 9 10 7 7 6 21/30 Write block 264 len 28672 data: 5 5 7 10 7 9 9 22/30 Flush 23/30 Write block 186 len 4096 data: 5 24/30 Write block 323 len 12288 data: 8 8 8 25/30 Read block 340 len 20480 26/30 Read block 250 len 28672 27/30 Write block 524 len 8192 data: 5 5 28/30 Write block 535 len 12288 data: 6 7 8 29/30 Write block 25 len 28672 data: 12 11 8 8 7 8 9 30/30 Write block 88 len 16384 data: 5 5 6 6 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:58.058540284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 155 155 155 F F F 1 020-039 88 88 88 155 155 155 T T T 2 040-059 90 90 90 160 160 160 F F F 3 060-079 89 89 89 157 157 157 F F F 4 080-099 84 84 84 144 144 144 T T T 5 100-119 90 90 90 159 159 159 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 90 90 90 159 159 159 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 90 90 90 160 160 160 T T T 10 200-219 88 88 88 155 155 155 F F F 11 220-239 90 90 90 158 158 158 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 90 90 90 160 160 160 F F F 14 280-299 90 90 90 160 160 160 F F F 15 300-319 90 90 90 160 160 160 F F F 16 320-339 90 90 90 160 160 160 T T T 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 90 90 90 159 159 159 F F F 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 89 89 89 157 157 157 F F F 25 500-519 90 90 90 160 160 160 F F F 26 520-539 89 89 89 157 157 157 T T T 27 540-559 79 79 79 133 133 133 F F F 28 560-579 89 89 89 157 157 157 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 90, Max flush: 160 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 155 155 155 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 90 90 90 160 160 160 F F F 3 060-079 89 89 89 157 157 157 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 90 90 90 159 159 159 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 90 90 90 159 159 159 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 91 91 91 161 161 161 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 90 90 90 158 158 158 F F F 12 240-259 83 83 83 141 141 141 F F F 13 260-279 90 90 90 160 160 160 F F F 14 280-299 90 90 90 160 160 160 F F F 15 300-319 90 90 90 160 160 160 F F F 16 320-339 91 91 91 161 161 161 F F F 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 90 90 90 159 159 159 F F F 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 89 89 89 157 157 157 F F F 25 500-519 90 90 90 160 160 160 F F F 26 520-539 91 91 91 161 161 161 F F F 27 540-559 79 79 79 133 133 133 F F F 28 560-579 89 89 89 157 157 157 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 91, Max flush: 161 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.357051986Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:02.398655871Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.399027759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.429649842Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.430193055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.430228315Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.430243866Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.430258476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550} {"msg":"Crucible e2c9c1a2-2a23-4542-93dc-ca6bce191cf1 has session id: 368bd551-6669-4753-8e20-327a51f30f70","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.430798469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Upstairs opts: Upstairs UUID: e2c9c1a2-2a23-4542-93dc-ca6bce191cf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.43084711Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Crucible stats registered with UUID: e2c9c1a2-2a23-4542-93dc-ca6bce191cf1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.430873631Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460386256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460413527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550} {"msg":"e2c9c1a2-2a23-4542-93dc-ca6bce191cf1 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460463328Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460491679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460517589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.46053265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.46054733Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.46056123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460574821Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460588291Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460610542Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460639052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_connection connected from Ok(127.0.0.1:61234)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460665573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_connection connected from Ok(127.0.0.1:51423)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460694204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_connection connected from Ok(127.0.0.1:34074)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.460725785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.461451842Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.461481512Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.461553894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.461627396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.461662937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.461693337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462120798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462152948Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462169519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462683961Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462729362Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462760093Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462782763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462807074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462830895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462853905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462881916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[0]R flush_numbers[0..12]: [155, 161, 160, 157, 161, 159, 160, 159, 148, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462907237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[0]R generation[0..12]: [88, 91, 90, 89, 91, 90, 90, 90, 86, 91, 88, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462933627Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.462975908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[1]R flush_numbers[0..12]: [155, 161, 160, 157, 161, 159, 160, 159, 148, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463004539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[1]R generation[0..12]: [88, 91, 90, 89, 91, 90, 90, 90, 86, 91, 88, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463032169Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.46306166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[2]R flush_numbers[0..12]: [155, 161, 160, 157, 161, 159, 160, 159, 148, 161, 155, 158]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:11:02.463092361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[2]R generation[0..12]: [88, 91, 90, 89, 91, 90, 90, 90, 86, 91, 88, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463128832Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463159623Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463189443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463218944Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463247985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463279975Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"":"downstairs","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463308766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:11:02.463335087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"0","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463366577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"1","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463382788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"client":"2","":"downstairs","session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463396548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463409538Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"e2c9c1a2-2a23-4542-93dc-ca6bce191cf1 is now active with session: 368bd551-6669-4753-8e20-327a51f30f70","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463422719Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463435039Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550,"session_id":"368bd551-6669-4753-8e20-327a51f30f70"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:02.463447319Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 314 len 32768 data: 7 8 9 9 7 9 7 8 02/30 Read block 586 len 20480 03/30 Write block 548 len 28672 data: 6 5 6 5 8 8 10 04/30 Read block 342 len 28672 05/30 Flush 06/30 Write block 252 len 20480 data: 8 8 8 9 9 07/30 Read block 352 len 20480 08/30 Read block 8 len 12288 09/30 Read block 507 len 12288 10/30 Write block 12 len 28672 data: 7 8 8 9 10 9 8 11/30 Read block 490 len 32768 12/30 Read block 42 len 20480 13/30 Read block 254 len 36864 14/30 Write block 269 len 32768 data: 10 10 8 9 10 7 8 6 15/30 Read block 464 len 36864 16/30 Read block 504 len 8192 17/30 Read block 235 len 36864 18/30 Write block 434 len 12288 data: 6 5 5 19/30 Write block 162 len 20480 data: 14 10 10 10 9 20/30 Write block 408 len 40960 data: 10 9 9 9 11 10 9 7 6 6 21/30 Write block 420 len 16384 data: 5 6 8 8 22/30 Read block 412 len 8192 23/30 Read block 333 len 12288 24/30 Read block 581 len 32768 25/30 Write block 151 len 12288 data: 4 6 10 26/30 Read block 30 len 40960 27/30 Write block 77 len 4096 data: 8 28/30 Read block 316 len 28672 29/30 Read block 534 len 36864 30/30 Write block 538 len 12288 data: 7 7 8 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 5 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:02.483696594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 155 155 155 T T T 1 020-039 91 91 91 161 161 161 F F F 2 040-059 90 90 90 160 160 160 F F F 3 060-079 89 89 89 157 157 157 T T T 4 080-099 91 91 91 161 161 161 F F F 5 100-119 90 90 90 159 159 159 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 90 90 90 159 159 159 T T T 8 160-179 86 86 86 148 148 148 T T T 9 180-199 91 91 91 161 161 161 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 90 90 90 158 158 158 F F F 12 240-259 83 83 83 141 141 141 T T T 13 260-279 90 90 90 160 160 160 T T T 14 280-299 90 90 90 160 160 160 F F F 15 300-319 92 92 92 162 162 162 F F F 16 320-339 92 92 92 162 162 162 F F F 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 82 82 82 139 139 139 T T T 21 420-439 90 90 90 159 159 159 T T T 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 89 89 89 157 157 157 F F F 25 500-519 90 90 90 160 160 160 F F F 26 520-539 91 91 91 161 161 161 T T T 27 540-559 92 92 92 162 162 162 T T T 28 560-579 89 89 89 157 157 157 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 92, Max flush: 162 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 90 90 90 160 160 160 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 90 90 90 159 159 159 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 93 93 93 163 163 163 F F F 8 160-179 93 93 93 163 163 163 F F F 9 180-199 91 91 91 161 161 161 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 90 90 90 158 158 158 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 93 93 93 163 163 163 F F F 14 280-299 90 90 90 160 160 160 F F F 15 300-319 92 92 92 162 162 162 F F F 16 320-339 92 92 92 162 162 162 F F F 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 86 86 86 148 148 148 F F F 20 400-419 93 93 93 163 163 163 F F F 21 420-439 93 93 93 163 163 163 F F F 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 89 89 89 157 157 157 F F F 25 500-519 90 90 90 160 160 160 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 93 93 93 163 163 163 F F F 28 560-579 89 89 89 157 157 157 F F F 29 580-599 80 80 80 134 134 134 F F F Max gen: 93, Max flush: 163 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.804070048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.846683126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.847302411Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.847338952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.847353942Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.847369183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558} {"msg":"Crucible 1f165a14-614f-431e-a4a0-af37f64c76c0 has session id: 0b04d175-54e8-40f5-aaeb-ceba01fe69c8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.847862624Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Upstairs opts: Upstairs UUID: 1f165a14-614f-431e-a4a0-af37f64c76c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.848008078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Crucible stats registered with UUID: 1f165a14-614f-431e-a4a0-af37f64c76c0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.848065429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.877916953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.877949874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558} {"msg":"1f165a14-614f-431e-a4a0-af37f64c76c0 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.877979384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878002865Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878050396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878164929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.87821479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878254221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878409074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878455356Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878490567Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878511657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41673)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878530208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56862)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878547868Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48180)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.878565518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.87906209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879104381Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879129522Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879146392Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879191693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879223894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879712716Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879743926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.879760477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880268879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.88030636Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.88032294Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880336021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880349891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880363071Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880378752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880392822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[0]R flush_numbers[0..12]: [163, 161, 160, 163, 161, 159, 160, 163, 163, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880406882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[0]R generation[0..12]: [93, 91, 90, 93, 91, 90, 90, 93, 93, 91, 88, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880420873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880435013Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[1]R flush_numbers[0..12]: [163, 161, 160, 163, 161, 159, 160, 163, 163, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880448893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[1]R generation[0..12]: [93, 91, 90, 93, 91, 90, 90, 93, 93, 91, 88, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880462764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880476934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[2]R flush_numbers[0..12]: [163, 161, 160, 163, 161, 159, 160, 163, 163, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880490954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"[2]R generation[0..12]: [93, 91, 90, 93, 91, 90, 90, 93, 93, 91, 88, 90]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880504735Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"Wait for a query_work_queue command to finish before sending IO } {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880529635Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880543165Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880556346Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880569416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880584357Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"":"downstairs","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880597967Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880610167Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"0","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880624587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"1","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880637988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"client":"2","":"downstairs","session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880651328Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880663879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"1f165a14-614f-431e-a4a0-af37f64c76c0 is now active with session: 0b04d175-54e8-40f5-aaeb-ceba01fe69c8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880676929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880689299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558,"session_id":"0b04d175-54e8-40f5-aaeb-ceba01fe69c8"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:06.880701759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 151 len 36864 data: 5 7 11 12 13 14 13 14 15 02/30 Write block 292 len 4096 data: 11 03/30 Write block 186 len 16384 data: 6 6 3 3 04/30 Read block 26 len 8192 05/30 Write block 116 len 8192 data: 6 6 06/30 Flush 07/30 Flush 08/30 Flush 09/30 Write block 542 len 4096 data: 8 10/30 Write block 382 len 40960 data: 8 6 7 6 7 8 6 7 7 6 11/30 Write block 594 len 20480 data: 5 5 5 4 3 12/30 Read block 118 len 40960 13/30 Flush 14/30 Flush 15/30 Write block 420 len 28672 data: 6 7 9 9 8 10 10 16/30 Read block 82 len 40960 17/30 Write block 225 len 24576 data: 10 8 8 6 7 7 18/30 Read block 433 len 32768 19/30 Write block 174 len 28672 data: 8 7 8 7 9 8 9 20/30 Write block 328 len 28672 data: 8 8 7 6 6 8 9 21/30 Flush 22/30 Write block 511 len 24576 data: 8 6 5 6 10 9 23/30 Read block 580 len 28672 24/30 Read block 131 len 4096 25/30 Flush 26/30 Write block 482 len 28672 data: 12 10 11 10 9 11 9 27/30 Write block 294 len 32768 data: 8 8 9 12 9 6 5 6 28/30 Read block 369 len 16384 29/30 Read block 232 len 32768 30/30 Write block 158 len 36864 data: 15 16 13 13 15 11 11 11 10 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:06.900254397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 90 90 90 160 160 160 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 94 94 94 164 164 164 T T T 8 160-179 94 94 94 169 169 169 T T T 9 180-199 94 94 94 169 169 169 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 94 94 94 169 169 169 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 93 93 93 163 163 163 F F F 14 280-299 94 94 94 164 164 164 T T T 15 300-319 92 92 92 162 162 162 T T T 16 320-339 94 94 94 169 169 169 F F F 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 94 94 94 167 167 167 F F F 20 400-419 93 93 93 163 163 163 F F F 21 420-439 94 94 94 169 169 169 F F F 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 89 89 89 157 157 157 T T T 25 500-519 94 94 94 170 170 170 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 94 94 94 167 167 167 F F F 28 560-579 89 89 89 157 157 157 F F F 29 580-599 94 94 94 167 167 167 F F F Max gen: 94, Max flush: 170 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 90 90 90 160 160 160 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 95 95 95 171 171 171 F F F 8 160-179 95 95 95 171 171 171 F F F 9 180-199 94 94 94 169 169 169 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 94 94 94 169 169 169 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 93 93 93 163 163 163 F F F 14 280-299 95 95 95 171 171 171 F F F 15 300-319 95 95 95 171 171 171 F F F 16 320-339 94 94 94 169 169 169 F F F 17 340-359 88 88 88 155 155 155 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 94 94 94 167 167 167 F F F 20 400-419 93 93 93 163 163 163 F F F 21 420-439 94 94 94 169 169 169 F F F 22 440-459 88 88 88 155 155 155 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 94 94 94 170 170 170 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 94 94 94 167 167 167 F F F 28 560-579 89 89 89 157 157 157 F F F 29 580-599 94 94 94 167 167 167 F F F Max gen: 95, Max flush: 171 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.166155129Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.209462214Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.210079369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.21011488Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.21012916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.21014369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566} {"msg":"Crucible 868646d5-bee0-4c1f-a6f5-61f771d47744 has session id: 73558de2-9761-4aab-a8af-dd23fe0bd269","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.210617562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Upstairs opts: Upstairs UUID: 868646d5-bee0-4c1f-a6f5-61f771d47744, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.210662843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Crucible stats registered with UUID: 868646d5-bee0-4c1f-a6f5-61f771d47744","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.210681683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240277331Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240310192Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566} {"msg":"868646d5-bee0-4c1f-a6f5-61f771d47744 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240369643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240397574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240428244Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240444355Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240458425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240472025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240485856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240499366Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240513586Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240527527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_connection connected from Ok(127.0.0.1:48416)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240545767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_connection connected from Ok(127.0.0.1:51081)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.2406534Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_connection connected from Ok(127.0.0.1:35244)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.240692971Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.241271184Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.241300455Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.241439048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.241459559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.241667784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.241696425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.24193762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242131305Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242436082Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242551945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242573675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242606686Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242663038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242682318Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242696069Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242929494Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242958185Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[0]R flush_numbers[0..12]: [163, 161, 160, 163, 161, 164, 160, 171, 171, 169, 155, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242974855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[0]R generation[0..12]: [93, 91, 90, 93, 91, 94, 90, 95, 95, 94, 88, 94]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.242989645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243008156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[1]R flush_numbers[0..12]: [163, 161, 160, 163, 161, 164, 160, 171, 171, 169, 155, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243023306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[1]R generation[0..12]: [93, 91, 90, 93, 91, 94, 90, 95, 95, 94, 88, 94]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243037277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243052417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[2]R flush_numbers[0..12]: [163, 161, 160, 163, 161, 164, 160, 171, 171, 169, 155, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243066427Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[2]R generation[0..12]: [93, 91, 90, 93, 91, 94, 90, 95, 95, 94, 88, 94]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243080148Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243094438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243107898Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243121299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-09T15:11:11.243134789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.24315686Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"":"downstairs","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.24317033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.24318244Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"0","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.24319587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"1","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243209201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"client":"2","":"downstairs","session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243224841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243237702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"868646d5-bee0-4c1f-a6f5-61f771d47744 is now active with session: 73558de2-9761-4aab-a8af-dd23fe0bd269","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243250792Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243263132Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566,"session_id":"73558de2-9761-4aab-a8af-dd23fe0bd269"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:11.243275562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 16384 data: 8 9 8 10 02/30 Write block 353 len 40960 data: 7 6 5 7 6 4 6 7 8 8 03/30 Read block 219 len 12288 04/30 Flush 05/30 Write block 572 len 4096 data: 10 06/30 Write block 40 len 28672 data: 7 9 11 11 9 9 11 07/30 Write block 586 len 16384 data: 7 7 5 7 08/30 Write block 56 len 12288 data: 14 10 10 09/30 Write block 284 len 36864 data: 8 9 12 13 10 10 9 11 12 10/30 Read block 385 len 40960 11/30 Read block 161 len 32768 12/30 Write block 495 len 12288 data: 7 6 5 13/30 Read block 477 len 12288 14/30 Write block 268 len 8192 data: 8 11 15/30 Write block 402 len 8192 data: 11 13 16/30 Read block 236 len 20480 17/30 Read block 398 len 32768 18/30 Read block 170 len 24576 19/30 Read block 297 len 28672 20/30 Write block 439 len 32768 data: 6 5 5 5 5 5 4 4 21/30 Read block 312 len 28672 22/30 Flush 23/30 Write block 503 len 20480 data: 9 8 7 9 10 24/30 Write block 510 len 8192 data: 10 9 25/30 Read block 165 len 4096 26/30 Write block 500 len 20480 data: 6 7 7 10 9 27/30 Write block 354 len 4096 data: 7 28/30 Read block 189 len 16384 29/30 Read block 274 len 8192 30/30 Write block 384 len 20480 data: 8 7 8 9 7 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:11.261484898Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 96 96 96 173 173 173 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 95 95 95 171 171 171 F F F 8 160-179 96 96 96 172 172 172 F F F 9 180-199 94 94 94 169 169 169 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 94 94 94 169 169 169 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 96 96 96 173 173 173 F F F 14 280-299 96 96 96 173 173 173 F F F 15 300-319 95 95 95 171 171 171 F F F 16 320-339 94 94 94 169 169 169 F F F 17 340-359 96 96 96 172 172 172 T T T 18 360-379 96 96 96 172 172 172 F F F 19 380-399 94 94 94 167 167 167 T T T 20 400-419 96 96 96 173 173 173 F F F 21 420-439 96 96 96 173 173 173 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 96 96 96 173 173 173 F F F 25 500-519 94 94 94 170 170 170 T T T 26 520-539 93 93 93 163 163 163 F F F 27 540-559 94 94 94 167 167 167 F F F 28 560-579 96 96 96 173 173 173 F F F 29 580-599 96 96 96 173 173 173 F F F Max gen: 96, Max flush: 173 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 96 96 96 173 173 173 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 95 95 95 171 171 171 F F F 8 160-179 96 96 96 172 172 172 F F F 9 180-199 94 94 94 169 169 169 F F F 10 200-219 88 88 88 155 155 155 F F F 11 220-239 94 94 94 169 169 169 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 96 96 96 173 173 173 F F F 14 280-299 96 96 96 173 173 173 F F F 15 300-319 95 95 95 171 171 171 F F F 16 320-339 94 94 94 169 169 169 F F F 17 340-359 97 97 97 174 174 174 F F F 18 360-379 96 96 96 172 172 172 F F F 19 380-399 97 97 97 174 174 174 F F F 20 400-419 96 96 96 173 173 173 F F F 21 420-439 96 96 96 173 173 173 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 96 96 96 173 173 173 F F F 25 500-519 97 97 97 174 174 174 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 94 94 94 167 167 167 F F F 28 560-579 96 96 96 173 173 173 F F F 29 580-599 96 96 96 173 173 173 F F F Max gen: 97, Max flush: 174 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.51484076Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.557547681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.558176016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.558213637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.558228367Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.558243397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574} {"msg":"Crucible 12be49d5-9020-4f21-975b-26d45a136672 has session id: 31b0af3d-42f6-4182-a6f5-cba2f221f5a5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.558667648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Upstairs opts: Upstairs UUID: 12be49d5-9020-4f21-975b-26d45a136672, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.558705459Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Crucible stats registered with UUID: 12be49d5-9020-4f21-975b-26d45a136672","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.558723439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588269915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588302706Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574} {"msg":"12be49d5-9020-4f21-975b-26d45a136672 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588371408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588394478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588424019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.58844231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588499571Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588585373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588617114Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588647444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588677825Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588704556Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:59146)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588752127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47749)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.58889793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39620)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.588936561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.589356171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.589383962Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.589398492Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.589412203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.589425743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.589444713Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590063308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590108649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.59014226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590664992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590697993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590713534Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590726764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590740714Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590754425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590766775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590780275Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[0]R flush_numbers[0..12]: [163, 161, 173, 163, 161, 164, 160, 171, 172, 169, 155, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590794176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[0]R generation[0..12]: [93, 91, 96, 93, 91, 94, 90, 95, 96, 94, 88, 94]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590808126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590822456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[1]R flush_numbers[0..12]: [163, 161, 173, 163, 161, 164, 160, 171, 172, 169, 155, 169]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590836317Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[1]R generation[0..12]: [93, 91, 96, 93, 91, 94, 90, 95, 96, 94, 88, 94]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590850017Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.590868148Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[2]R flush_numbers[0..12]: [163, 161, 173, 163, 161, 164, 160, 171, 172, 169, 155, 169]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-09T15:11:15.590882748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[2]R generation[0..12]: [93, 91, 96, 93, 91, 94, 90, 95, 96, 94, 88, 94]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.59095847Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.59097533Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.59098851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591001931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591015251Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591028311Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"":"downstairs","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591041741Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591053862Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"0","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591067392Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"1","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591080802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"client":"2","":"downstairs","session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591094183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591106553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"12be49d5-9020-4f21-975b-26d45a136672 is now active with session: 31b0af3d-42f6-4182-a6f5-cba2f221f5a5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591119594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591135234Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574,"session_id":"31b0af3d-42f6-4182-a6f5-cba2f221f5a5"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:15.591148164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 566 len 28672 data: 4 6 6 9 10 10 11 02/30 Read block 530 len 32768 03/30 Read block 415 len 36864 04/30 Read block 370 len 4096 05/30 Read block 512 len 32768 06/30 Read block 443 len 32768 07/30 Flush 08/30 Write block 292 len 24576 data: 13 9 9 9 10 13 09/30 Write block 183 len 12288 data: 6 5 6 10/30 Read block 343 len 20480 11/30 Write block 149 len 32768 data: 6 6 6 8 12 13 14 15 12/30 Read block 179 len 16384 13/30 Read block 105 len 4096 14/30 Write block 218 len 24576 data: 6 7 8 9 11 11 15/30 Write block 335 len 4096 data: 9 16/30 Read block 577 len 8192 17/30 Read block 408 len 32768 18/30 Flush 19/30 Write block 358 len 12288 data: 5 7 8 20/30 Read block 274 len 40960 21/30 Write block 415 len 8192 data: 8 7 22/30 Flush 23/30 Write block 390 len 24576 data: 8 7 6 6 7 8 24/30 Write block 346 len 36864 data: 14 15 11 11 9 8 9 8 8 25/30 Flush 26/30 Write block 318 len 12288 data: 8 10 8 27/30 Write block 543 len 24576 data: 10 10 8 6 6 7 28/30 Write block 550 len 12288 data: 7 6 9 29/30 Write block 236 len 12288 data: 8 8 8 30/30 Read block 479 len 16384 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:15.609424181Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 96 96 96 173 173 173 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 98 98 98 176 176 176 F F F 8 160-179 96 96 96 172 172 172 F F F 9 180-199 98 98 98 176 176 176 F F F 10 200-219 98 98 98 176 176 176 F F F 11 220-239 98 98 98 176 176 176 T T T 12 240-259 93 93 93 163 163 163 F F F 13 260-279 96 96 96 173 173 173 F F F 14 280-299 98 98 98 176 176 176 F F F 15 300-319 95 95 95 171 171 171 T T T 16 320-339 98 98 98 176 176 176 T T T 17 340-359 98 98 98 178 178 178 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 96 96 96 173 173 173 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 96 96 96 173 173 173 F F F 25 500-519 97 97 97 174 174 174 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 94 94 94 167 167 167 T T T 28 560-579 98 98 98 175 175 175 F F F 29 580-599 96 96 96 173 173 173 F F F Max gen: 98, Max flush: 178 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 96 96 96 173 173 173 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 98 98 98 176 176 176 F F F 8 160-179 96 96 96 172 172 172 F F F 9 180-199 98 98 98 176 176 176 F F F 10 200-219 98 98 98 176 176 176 F F F 11 220-239 99 99 99 179 179 179 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 96 96 96 173 173 173 F F F 14 280-299 98 98 98 176 176 176 F F F 15 300-319 99 99 99 179 179 179 F F F 16 320-339 99 99 99 179 179 179 F F F 17 340-359 98 98 98 178 178 178 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 96 96 96 173 173 173 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 96 96 96 173 173 173 F F F 25 500-519 97 97 97 174 174 174 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 99 99 99 179 179 179 F F F 28 560-579 98 98 98 175 175 175 F F F 29 580-599 96 96 96 173 173 173 F F F Max gen: 99, Max flush: 179 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.867145107Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:19.908636799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.909023098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.939358183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.939917067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.939952548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.939971428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.939986778Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582} {"msg":"Crucible d7ca7c65-cbae-459d-93ad-3d57c9106687 has session id: f7301f7f-562b-41eb-8eb4-d8626aee70ef","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.940395478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Upstairs opts: Upstairs UUID: d7ca7c65-cbae-459d-93ad-3d57c9106687, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.94044228Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Crucible stats registered with UUID: d7ca7c65-cbae-459d-93ad-3d57c9106687","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.94046931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970331604Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970397165Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582} {"msg":"d7ca7c65-cbae-459d-93ad-3d57c9106687 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970428716Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970443647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970467647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970482247Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970496548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970510528Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.97057243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970614601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970644962Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970672472Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:46413)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970700573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:59183)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970729084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:52146)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.970928058Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.971366339Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.97140773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.971487991Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.971519952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.971537133Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.971552403Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972067865Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972126847Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972214589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972638119Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.97268922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972705321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972730341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972745552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972759202Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972771502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972785303Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[0]R flush_numbers[0..12]: [163, 161, 173, 163, 161, 164, 160, 176, 172, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972799443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[0]R generation[0..12]: [93, 91, 96, 93, 91, 94, 90, 98, 96, 98, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972813373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972827744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[1]R flush_numbers[0..12]: [163, 161, 173, 163, 161, 164, 160, 176, 172, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972841654Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[1]R generation[0..12]: [93, 91, 96, 93, 91, 94, 90, 98, 96, 98, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972855434Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972869475Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[2]R flush_numbers[0..12]: [163, 161, 173, 163, 161, 164, 160, 176, 172, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972883565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[2]R generation[0..12]: [93, 91, 96, 93, 91, 94, 90, 98, 96, 98, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972911846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972941916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.972966457Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973002898Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973030899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973058199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"":"downstairs","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.97308504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.97311048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"0","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973129691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"1","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973149471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"client":"2","":"downstairs","session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2024-12-09T15:11:19.973179122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973201843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"d7ca7c65-cbae-459d-93ad-3d57c9106687 is now active with session: f7301f7f-562b-41eb-8eb4-d8626aee70ef","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973224783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973248044Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582,"session_id":"f7301f7f-562b-41eb-8eb4-d8626aee70ef"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:19.973270424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 12288 data: 11 12 11 02/30 Write block 558 len 36864 data: 8 5 3 3 5 5 5 5 5 03/30 Read block 487 len 36864 04/30 Read block 82 len 8192 05/30 Write block 318 len 36864 data: 9 11 9 9 7 9 9 9 10 06/30 Write block 43 len 32768 data: 12 10 10 12 10 8 10 11 07/30 Read block 21 len 8192 08/30 Read block 202 len 28672 09/30 Read block 282 len 32768 10/30 Write block 368 len 36864 data: 11 10 6 5 5 4 4 3 5 11/30 Read block 89 len 12288 12/30 Write block 563 len 4096 data: 6 13/30 Read block 346 len 24576 14/30 Read block 334 len 36864 15/30 Flush 16/30 Write block 182 len 40960 data: 6 7 6 7 7 7 4 4 3 6 17/30 Read block 353 len 16384 18/30 Read block 476 len 12288 19/30 Write block 50 len 8192 data: 12 8 20/30 Flush 21/30 Write block 140 len 20480 data: 9 12 10 10 10 22/30 Read block 523 len 32768 23/30 Write block 501 len 32768 data: 8 8 11 10 8 10 11 10 24/30 Read block 150 len 16384 25/30 Write block 54 len 16384 data: 11 14 15 11 26/30 Read block 551 len 8192 27/30 Read block 137 len 12288 28/30 Write block 505 len 12288 data: 9 11 12 29/30 Write block 555 len 36864 data: 8 8 11 9 6 4 4 6 7 30/30 Read block 331 len 12288 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 10 8 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:19.993424756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 100 100 100 181 181 181 T T T 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 98 98 98 176 176 176 T T T 8 160-179 96 96 96 172 172 172 F F F 9 180-199 100 100 100 181 181 181 F F F 10 200-219 98 98 98 176 176 176 F F F 11 220-239 99 99 99 179 179 179 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 96 96 96 173 173 173 F F F 14 280-299 98 98 98 176 176 176 F F F 15 300-319 100 100 100 180 180 180 F F F 16 320-339 100 100 100 180 180 180 F F F 17 340-359 98 98 98 178 178 178 F F F 18 360-379 100 100 100 180 180 180 F F F 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 96 96 96 173 173 173 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 100 100 100 180 180 180 F F F 25 500-519 97 97 97 174 174 174 T T T 26 520-539 93 93 93 163 163 163 F F F 27 540-559 100 100 100 180 180 180 T T T 28 560-579 100 100 100 180 180 180 T T T 29 580-599 96 96 96 173 173 173 F F F Max gen: 100, Max flush: 181 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 101 101 101 182 182 182 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 101 101 101 182 182 182 F F F 8 160-179 96 96 96 172 172 172 F F F 9 180-199 100 100 100 181 181 181 F F F 10 200-219 98 98 98 176 176 176 F F F 11 220-239 99 99 99 179 179 179 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 96 96 96 173 173 173 F F F 14 280-299 98 98 98 176 176 176 F F F 15 300-319 100 100 100 180 180 180 F F F 16 320-339 100 100 100 180 180 180 F F F 17 340-359 98 98 98 178 178 178 F F F 18 360-379 100 100 100 180 180 180 F F F 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 96 96 96 173 173 173 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 100 100 100 180 180 180 F F F 25 500-519 101 101 101 182 182 182 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 101 101 101 182 182 182 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 96 96 96 173 173 173 F F F Max gen: 101, Max flush: 182 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.286754134Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.330210883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.330806567Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.330836768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.330853198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.330879069Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590} {"msg":"Crucible 51f50eb5-251b-4dbb-86df-836cc41d1a08 has session id: 57640d94-63e1-42b6-bb8c-5932a65f63d1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.331261758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Upstairs opts: Upstairs UUID: 51f50eb5-251b-4dbb-86df-836cc41d1a08, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.331305729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Crucible stats registered with UUID: 51f50eb5-251b-4dbb-86df-836cc41d1a08","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.3313296Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361694645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361728736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590} {"msg":"51f50eb5-251b-4dbb-86df-836cc41d1a08 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361763247Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361780397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361809358Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361827649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361845349Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361862439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.36187977Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.36189673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361914191Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361931681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58466)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.361980162Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39157)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.362160786Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61697)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.362211308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.362768391Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.362810202Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.362955565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.363022157Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.363098969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.36314051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.363468398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.363497969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.363776675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.363999961Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364028851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364043922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364106673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364127744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364141684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364290617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364310098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[0]R flush_numbers[0..12]: [163, 161, 182, 163, 161, 164, 160, 182, 172, 181, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364324468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[0]R generation[0..12]: [93, 91, 101, 93, 91, 94, 90, 101, 96, 100, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364338429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364352849Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[1]R flush_numbers[0..12]: [163, 161, 182, 163, 161, 164, 160, 182, 172, 181, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364366989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[1]R generation[0..12]: [93, 91, 101, 93, 91, 94, 90, 101, 96, 100, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.36438121Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.36439552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[2]R flush_numbers[0..12]: [163, 161, 182, 163, 161, 164, 160, 182, 172, 181, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.36440951Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[2]R generation[0..12]: [93, 91, 101, 93, 91, 94, 90, 101, 96, 100, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.36442326Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364437141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364453371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364467361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364481302Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:11:24.364494482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"":"downstairs","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364512763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364524973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"0","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364538423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"1","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364551724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"client":"2","":"downstairs","session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364565054Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364577494Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"51f50eb5-251b-4dbb-86df-836cc41d1a08 is now active with session: 57640d94-63e1-42b6-bb8c-5932a65f63d1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364590415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364602505Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590,"session_id":"57640d94-63e1-42b6-bb8c-5932a65f63d1"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:24.364614955Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Flush 04/30 Write block 512 len 40960 data: 7 6 7 11 10 10 8 6 8 7 05/30 Write block 34 len 32768 data: 8 7 8 9 9 8 8 10 06/30 Read block 143 len 24576 07/30 Write block 587 len 32768 data: 8 6 8 6 6 6 5 6 08/30 Write block 427 len 32768 data: 10 11 10 10 8 8 7 7 09/30 Write block 450 len 20480 data: 5 6 6 7 7 10/30 Write block 50 len 16384 data: 13 9 10 9 11/30 Read block 496 len 4096 12/30 Write block 50 len 16384 data: 14 10 11 10 13/30 Read block 307 len 12288 14/30 Read block 291 len 4096 15/30 Read block 322 len 36864 16/30 Read block 195 len 24576 17/30 Read block 470 len 24576 18/30 Write block 303 len 4096 data: 6 19/30 Write block 491 len 36864 data: 8 6 6 8 8 7 6 5 6 20/30 Write block 366 len 4096 data: 10 21/30 Write block 233 len 36864 data: 12 9 8 9 9 9 8 6 7 22/30 Write block 422 len 40960 data: 10 10 9 11 11 11 12 11 11 9 23/30 Read block 408 len 40960 24/30 Write block 168 len 4096 data: 4 25/30 Read block 419 len 16384 26/30 Write block 550 len 12288 data: 8 7 10 27/30 Read block 18 len 28672 28/30 Read block 194 len 16384 29/30 Read block 196 len 4096 30/30 Write block 268 len 28672 data: 9 12 11 9 10 11 8 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:24.382912222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 91 91 91 161 161 161 T T T 2 040-059 101 101 101 182 182 182 T T T 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 101 101 101 182 182 182 F F F 8 160-179 96 96 96 172 172 172 T T T 9 180-199 100 100 100 181 181 181 F F F 10 200-219 98 98 98 176 176 176 F F F 11 220-239 99 99 99 179 179 179 T T T 12 240-259 93 93 93 163 163 163 T T T 13 260-279 96 96 96 173 173 173 T T T 14 280-299 98 98 98 176 176 176 F F F 15 300-319 100 100 100 180 180 180 T T T 16 320-339 100 100 100 180 180 180 F F F 17 340-359 98 98 98 178 178 178 F F F 18 360-379 100 100 100 180 180 180 T T T 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 96 96 96 173 173 173 T T T 22 440-459 96 96 96 173 173 173 T T T 23 460-479 80 80 80 137 137 137 F F F 24 480-499 100 100 100 180 180 180 T T T 25 500-519 101 101 101 182 182 182 T T T 26 520-539 93 93 93 163 163 163 T T T 27 540-559 101 101 101 182 182 182 T T T 28 560-579 101 101 101 182 182 182 F F F 29 580-599 96 96 96 173 173 173 T T T Max gen: 101, Max flush: 182 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 163 163 163 F F F 1 020-039 102 102 102 183 183 183 F F F 2 040-059 102 102 102 183 183 183 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 90 90 90 160 160 160 F F F 7 140-159 101 101 101 182 182 182 F F F 8 160-179 102 102 102 183 183 183 F F F 9 180-199 100 100 100 181 181 181 F F F 10 200-219 98 98 98 176 176 176 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 102 102 102 183 183 183 F F F 13 260-279 102 102 102 183 183 183 F F F 14 280-299 98 98 98 176 176 176 F F F 15 300-319 102 102 102 183 183 183 F F F 16 320-339 100 100 100 180 180 180 F F F 17 340-359 98 98 98 178 178 178 F F F 18 360-379 102 102 102 183 183 183 F F F 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 102 102 102 183 183 183 F F F 23 460-479 80 80 80 137 137 137 F F F 24 480-499 102 102 102 183 183 183 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 102 102 102 183 183 183 F F F 27 540-559 102 102 102 183 183 183 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 102, Max flush: 183 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.70384412Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.747299869Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.747922954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.747959805Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.747974785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.747993205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598} {"msg":"Crucible 6f51ac54-f888-4f8d-adc7-9db6f79d1783 has session id: 33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.748464827Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Upstairs opts: Upstairs UUID: 6f51ac54-f888-4f8d-adc7-9db6f79d1783, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.748509178Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Crucible stats registered with UUID: 6f51ac54-f888-4f8d-adc7-9db6f79d1783","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.748534889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.778856023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.778889883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598} {"msg":"6f51ac54-f888-4f8d-adc7-9db6f79d1783 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.778941325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.778962525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.778991096Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779009016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779026087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779042617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779059498Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779076208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779093839Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779115859Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44311)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779225162Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49102)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779273813Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37574)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.779412196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.78001179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.780040331Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.780056231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.780070462Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.780115423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.780146924Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.780654786Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.780799109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.78083537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781156978Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781199659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.78123373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781262Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781291911Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781320462Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781382253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781411514Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[0]R flush_numbers[0..12]: [163, 183, 183, 163, 161, 164, 160, 182, 183, 181, 176, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781429964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[0]R generation[0..12]: [93, 102, 102, 93, 91, 94, 90, 101, 102, 100, 98, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781445665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781460455Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[1]R flush_numbers[0..12]: [163, 183, 183, 163, 161, 164, 160, 182, 183, 181, 176, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781474665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[1]R generation[0..12]: [93, 102, 102, 93, 91, 94, 90, 101, 102, 100, 98, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781488516Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781502626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[2]R flush_numbers[0..12]: [163, 183, 183, 163, 161, 164, 160, 182, 183, 181, 176, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781516516Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[2]R generation[0..12]: [93, 102, 102, 93, 91, 94, 90, 101, 102, 100, 98, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781530497Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781548857Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Max found gen is 103","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781563207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Generation requested: 104 >= found:103","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:11:28.781576828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781597658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781610899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"":"downstairs","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781624299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781636369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"0","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.78164974Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"1","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.78166329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"client":"2","":"downstairs","session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.78167668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781689261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"6f51ac54-f888-4f8d-adc7-9db6f79d1783 is now active with session: 33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781702211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781714391Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598,"session_id":"33a7b3d1-84ee-4c36-b87a-dddaad7dd6d6"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:28.781726751Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 4096 data: 3 02/30 Write block 337 len 24576 data: 7 8 7 7 7 6 03/30 Read block 167 len 32768 04/30 Read block 258 len 20480 05/30 Write block 139 len 28672 data: 6 10 13 11 11 11 9 06/30 Write block 154 len 8192 data: 14 15 07/30 Read block 491 len 20480 08/30 Read block 428 len 12288 09/30 Write block 254 len 8192 data: 9 10 10/30 Read block 463 len 28672 11/30 Read block 54 len 28672 12/30 Write block 355 len 32768 data: 6 8 7 6 8 9 9 9 13/30 Flush 14/30 Flush 15/30 Write block 464 len 12288 data: 7 7 7 16/30 Read block 410 len 32768 17/30 Write block 535 len 28672 data: 7 8 9 8 8 9 7 18/30 Write block 30 len 28672 data: 9 10 9 8 9 8 9 19/30 Read block 585 len 28672 20/30 Read block 255 len 16384 21/30 Write block 81 len 4096 data: 7 22/30 Read block 483 len 40960 23/30 Read block 344 len 8192 24/30 Read block 166 len 4096 25/30 Read block 51 len 32768 26/30 Read block 210 len 16384 27/30 Read block 539 len 12288 28/30 Write block 205 len 16384 data: 6 6 7 7 29/30 Write block 263 len 40960 data: 5 6 6 8 11 10 13 12 10 11 30/30 Write block 456 len 36864 data: 6 4 5 6 5 5 6 8 8 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:28.800939511Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 102 102 102 183 183 183 T T T 2 040-059 102 102 102 183 183 183 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 91 91 91 161 161 161 T T T 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 104 104 104 184 184 184 F F F 8 160-179 102 102 102 183 183 183 F F F 9 180-199 100 100 100 181 181 181 F F F 10 200-219 98 98 98 176 176 176 T T T 11 220-239 102 102 102 183 183 183 F F F 12 240-259 104 104 104 184 184 184 F F F 13 260-279 102 102 102 183 183 183 T T T 14 280-299 98 98 98 176 176 176 F F F 15 300-319 102 102 102 183 183 183 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 104 104 104 184 184 184 F F F 18 360-379 104 104 104 184 184 184 F F F 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 102 102 102 183 183 183 T T T 23 460-479 80 80 80 137 137 137 T T T 24 480-499 102 102 102 183 183 183 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 102 102 102 183 183 183 T T T 27 540-559 102 102 102 183 183 183 T T T 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 104, Max flush: 184 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 105 105 105 185 185 185 F F F 2 040-059 102 102 102 183 183 183 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 104 104 104 184 184 184 F F F 8 160-179 102 102 102 183 183 183 F F F 9 180-199 100 100 100 181 181 181 F F F 10 200-219 105 105 105 185 185 185 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 104 104 104 184 184 184 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 98 98 98 176 176 176 F F F 15 300-319 102 102 102 183 183 183 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 104 104 104 184 184 184 F F F 18 360-379 104 104 104 184 184 184 F F F 19 380-399 98 98 98 178 178 178 F F F 20 400-419 98 98 98 177 177 177 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 105 105 105 185 185 185 F F F 23 460-479 105 105 105 185 185 185 F F F 24 480-499 102 102 102 183 183 183 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 105 105 105 185 185 185 F F F 27 540-559 105 105 105 185 185 185 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 105, Max flush: 185 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.081007302Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.124942742Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.125611608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.125648949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.125663699Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.125678089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606} {"msg":"Crucible 52861cd8-44e4-4b8a-afa3-547719d277de has session id: e20d6184-b488-4d93-a6e3-c28d65870366","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.126064679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Upstairs opts: Upstairs UUID: 52861cd8-44e4-4b8a-afa3-547719d277de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.12610923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Crucible stats registered with UUID: 52861cd8-44e4-4b8a-afa3-547719d277de","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.12613437Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156010865Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156045075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606} {"msg":"52861cd8-44e4-4b8a-afa3-547719d277de active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156115117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156138328Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156167968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156195319Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.15623047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.15626418Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156299581Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156321852Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156356463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156394164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_connection connected from Ok(127.0.0.1:37737)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156429755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_connection connected from Ok(127.0.0.1:45032)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156511426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_connection connected from Ok(127.0.0.1:59890)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.156558818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157160742Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157192503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157209293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157226014Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157252354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157280115Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157883089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.1579202Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.157951591Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158467983Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158496914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158511954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158524445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158542295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158556575Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158569056Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158582736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[0]R flush_numbers[0..12]: [184, 185, 183, 163, 185, 164, 184, 184, 183, 181, 185, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158596866Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[0]R generation[0..12]: [104, 105, 102, 93, 105, 94, 104, 104, 102, 100, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158610957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158625127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[1]R flush_numbers[0..12]: [184, 185, 183, 163, 185, 164, 184, 184, 183, 181, 185, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158639067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[1]R generation[0..12]: [104, 105, 102, 93, 105, 94, 104, 104, 102, 100, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158652928Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158666818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[2]R flush_numbers[0..12]: [184, 185, 183, 163, 185, 164, 184, 184, 183, 181, 185, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158680568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[2]R generation[0..12]: [104, 105, 102, 93, 105, 94, 104, 104, 102, 100, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158694299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158708379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158721599Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.15873675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:11:33.15875019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158769351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"":"downstairs","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158782871Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158795231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"0","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158808592Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"1","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158821882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"client":"2","":"downstairs","session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158835912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158848402Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"52861cd8-44e4-4b8a-afa3-547719d277de is now active with session: e20d6184-b488-4d93-a6e3-c28d65870366","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158861053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158873123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606,"session_id":"e20d6184-b488-4d93-a6e3-c28d65870366"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:33.158885383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 50 len 24576 data: 15 11 12 11 12 15 02/30 Read block 484 len 8192 03/30 Read block 373 len 8192 04/30 Read block 371 len 40960 05/30 Read block 8 len 28672 06/30 Read block 58 len 20480 07/30 Read block 461 len 36864 08/30 Write block 195 len 28672 data: 9 8 10 8 8 7 7 09/30 Flush 10/30 Flush 11/30 Write block 445 len 40960 data: 5 5 4 4 5 6 7 7 8 8 12/30 Read block 136 len 16384 13/30 Read block 97 len 28672 14/30 Write block 387 len 16384 data: 10 8 8 9 15/30 Read block 507 len 4096 16/30 Write block 363 len 12288 data: 10 9 9 17/30 Write block 533 len 36864 data: 7 7 8 9 10 9 9 10 8 18/30 Read block 66 len 40960 19/30 Read block 225 len 36864 20/30 Read block 50 len 40960 21/30 Read block 105 len 16384 22/30 Read block 228 len 12288 23/30 Write block 417 len 8192 data: 7 5 24/30 Write block 173 len 8192 data: 9 9 25/30 Read block 368 len 8192 26/30 Write block 195 len 20480 data: 10 9 11 9 9 27/30 Read block 70 len 40960 28/30 Write block 293 len 12288 data: 10 10 10 29/30 Write block 45 len 36864 data: 11 13 11 9 11 16 12 13 12 30/30 Write block 19 len 32768 data: 8 10 10 11 10 10 13 12 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 10 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:33.178985574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 T T T 1 020-039 105 105 105 185 185 185 T T T 2 040-059 106 106 106 186 186 186 T T T 3 060-079 93 93 93 163 163 163 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 104 104 104 184 184 184 F F F 8 160-179 102 102 102 183 183 183 T T T 9 180-199 106 106 106 186 186 186 T T T 10 200-219 106 106 106 186 186 186 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 104 104 104 184 184 184 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 98 98 98 176 176 176 T T T 15 300-319 102 102 102 183 183 183 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 104 104 104 184 184 184 F F F 18 360-379 104 104 104 184 184 184 T T T 19 380-399 98 98 98 178 178 178 T T T 20 400-419 98 98 98 177 177 177 T T T 21 420-439 102 102 102 183 183 183 F F F 22 440-459 105 105 105 185 185 185 T T T 23 460-479 105 105 105 185 185 185 F F F 24 480-499 102 102 102 183 183 183 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 105 105 105 185 185 185 T T T 27 540-559 105 105 105 185 185 185 T T T 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 106, Max flush: 186 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 187 187 187 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 107 107 107 187 187 187 F F F 3 060-079 93 93 93 163 163 163 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 104 104 104 184 184 184 F F F 8 160-179 107 107 107 187 187 187 F F F 9 180-199 107 107 107 187 187 187 F F F 10 200-219 106 106 106 186 186 186 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 104 104 104 184 184 184 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 102 102 102 183 183 183 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 104 104 104 184 184 184 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 107 107 107 187 187 187 F F F 20 400-419 107 107 107 187 187 187 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 107 107 107 187 187 187 F F F 23 460-479 105 105 105 185 185 185 F F F 24 480-499 102 102 102 183 183 183 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 107 107 107 187 187 187 F F F 27 540-559 107 107 107 187 187 187 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 107, Max flush: 187 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.476264876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.518788813Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.519426738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.519463769Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.519478549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.519494439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614} {"msg":"Crucible afb61f26-7acc-4fd7-ad4e-a7d432fac422 has session id: 91629c0b-5966-4cd1-855f-d0b8d4cb92f9","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.519869988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Upstairs opts: Upstairs UUID: afb61f26-7acc-4fd7-ad4e-a7d432fac422, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.51991876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Crucible stats registered with UUID: afb61f26-7acc-4fd7-ad4e-a7d432fac422","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.51993932Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549730552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549763453Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614} {"msg":"afb61f26-7acc-4fd7-ad4e-a7d432fac422 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549813274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549833964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549862185Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549881106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549899126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549925777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549957718Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.549982608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550009589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550037199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60838)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.55006425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46005)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550096431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40905)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550271075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550740066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550768907Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550943231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.550976582Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.551012983Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.551031123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.551420702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.551452033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.551602707Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552055098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552086908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552103509Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552116949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552134179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.55214844Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.55216132Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.55217526Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[0]R flush_numbers[0..12]: [187, 187, 187, 163, 185, 164, 184, 184, 187, 187, 186, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552189481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[0]R generation[0..12]: [107, 107, 107, 93, 105, 94, 104, 104, 107, 107, 106, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552203611Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552218081Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[1]R flush_numbers[0..12]: [187, 187, 187, 163, 185, 164, 184, 184, 187, 187, 186, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552231962Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[1]R generation[0..12]: [107, 107, 107, 93, 105, 94, 104, 104, 107, 107, 106, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552245812Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552259973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[2]R flush_numbers[0..12]: [187, 187, 187, 163, 185, 164, 184, 184, 187, 187, 186, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552273943Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[2]R generation[0..12]: [107, 107, 107, 93, 105, 94, 104, 104, 107, 107, 106, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552287623Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552301594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552314684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552339264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552352485Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552365655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"":"downstairs","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552378845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552391026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"0","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552404206Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"1","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552417566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"client":"2","":"downstairs","session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552430917Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552443497Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"afb61f26-7acc-4fd7-ad4e-a7d432fac422 is now active with session: 91629c0b-5966-4cd1-855f-d0b8d4cb92f9","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552456507Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552468698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614,"session_id":"91629c0b-5966-4cd1-855f-d0b8d4cb92f9"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:37.552481098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 24576 data: 7 6 5 5 4 3 02/30 Read block 486 len 12288 03/30 Read block 469 len 28672 04/30 Flush 05/30 Write block 30 len 16384 data: 10 11 10 9 06/30 Read block 467 len 24576 07/30 Write block 312 len 24576 data: 5 7 8 9 10 10 08/30 Write block 154 len 40960 data: 15 16 16 14 16 17 14 14 16 12 09/30 Read block 529 len 12288 10/30 Flush 11/30 Write block 179 len 40960 data: 9 10 7 7 8 7 8 8 8 5 12/30 Write block 348 len 32768 data: 12 12 10 9 10 9 9 7 13/30 Write block 402 len 32768 data: 12 14 12 9 7 10 11 10 14/30 Read block 6 len 20480 15/30 Read block 420 len 28672 16/30 Write block 199 len 12288 data: 10 8 8 17/30 Write block 545 len 12288 data: 9 7 7 18/30 Read block 9 len 24576 19/30 Read block 16 len 24576 20/30 Write block 65 len 32768 data: 14 15 14 10 9 8 6 5 21/30 Write block 462 len 8192 data: 7 9 22/30 Read block 370 len 24576 23/30 Read block 134 len 36864 24/30 Flush 25/30 Write block 3 len 12288 data: 5 7 6 26/30 Read block 320 len 8192 27/30 Read block 249 len 4096 28/30 Read block 342 len 12288 29/30 Read block 400 len 36864 30/30 Read block 524 len 8192 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:37.572384884Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 187 187 187 T T T 1 020-039 108 108 108 189 189 189 F F F 2 040-059 107 107 107 187 187 187 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 108 108 108 189 189 189 F F F 8 160-179 108 108 108 190 190 190 F F F 9 180-199 108 108 108 190 190 190 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 108 108 108 188 188 188 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 108 108 108 189 189 189 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 108 108 108 190 190 190 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 107 107 107 187 187 187 F F F 20 400-419 108 108 108 190 190 190 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 107 107 107 187 187 187 F F F 23 460-479 108 108 108 190 190 190 F F F 24 480-499 102 102 102 183 183 183 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 107 107 107 187 187 187 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 108, Max flush: 190 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 108 108 108 189 189 189 F F F 2 040-059 107 107 107 187 187 187 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 108 108 108 189 189 189 F F F 8 160-179 108 108 108 190 190 190 F F F 9 180-199 108 108 108 190 190 190 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 108 108 108 188 188 188 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 108 108 108 189 189 189 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 108 108 108 190 190 190 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 107 107 107 187 187 187 F F F 20 400-419 108 108 108 190 190 190 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 107 107 107 187 187 187 F F F 23 460-479 108 108 108 190 190 190 F F F 24 480-499 102 102 102 183 183 183 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 107 107 107 187 187 187 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 109, Max flush: 191 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.812040258Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.854641036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.855257051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.855291092Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.855305962Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.855321243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622} {"msg":"Crucible d2c61c27-c283-4703-b55c-c16c72a7a8d8 has session id: 4488d0f6-b78b-4603-8816-c65bae200920","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.855858446Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Upstairs opts: Upstairs UUID: d2c61c27-c283-4703-b55c-c16c72a7a8d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.855893336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Crucible stats registered with UUID: d2c61c27-c283-4703-b55c-c16c72a7a8d8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.855913367Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886536409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88657384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622} {"msg":"d2c61c27-c283-4703-b55c-c16c72a7a8d8 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88659913Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886616751Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886646522Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886665422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886683052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886700053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886718293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886735504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886757404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886785715Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_connection connected from Ok(127.0.0.1:53171)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886841166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_connection connected from Ok(127.0.0.1:34279)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.886878177Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_connection connected from Ok(127.0.0.1:45960)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88702082Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.887652186Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.887690967Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.887729097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.887766098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.887821199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88783993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888328762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888360132Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888378363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888854604Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888884645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888900925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888967197Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.888987127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889016268Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889030228Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889044789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[0]R flush_numbers[0..12]: [191, 189, 187, 190, 185, 164, 184, 189, 190, 190, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889062619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[0]R generation[0..12]: [109, 108, 107, 108, 105, 94, 104, 108, 108, 108, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88907722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88909151Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[1]R flush_numbers[0..12]: [191, 189, 187, 190, 185, 164, 184, 189, 190, 190, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88910543Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[1]R generation[0..12]: [109, 108, 107, 108, 105, 94, 104, 108, 108, 108, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.88911915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889133081Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[2]R flush_numbers[0..12]: [191, 189, 187, 190, 185, 164, 184, 189, 190, 190, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889146931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[2]R generation[0..12]: [109, 108, 107, 108, 105, 94, 104, 108, 108, 108, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889160811Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889174882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889188012Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:11:41.889201263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889234813Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889248374Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"":"downstairs","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889261954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889276284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"0","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889290155Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"1","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889303485Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"client":"2","":"downstairs","session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889316945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889329256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"d2c61c27-c283-4703-b55c-c16c72a7a8d8 is now active with session: 4488d0f6-b78b-4603-8816-c65bae200920","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889342086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889354046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622,"session_id":"4488d0f6-b78b-4603-8816-c65bae200920"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:41.889366476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 16384 data: 10 11 8 8 02/30 Read block 383 len 24576 03/30 Write block 245 len 40960 data: 6 5 4 3 5 6 6 9 9 10 04/30 Flush 05/30 Write block 438 len 28672 data: 4 7 6 6 6 6 6 06/30 Read block 478 len 12288 07/30 Read block 282 len 28672 08/30 Read block 315 len 24576 09/30 Read block 239 len 8192 10/30 Read block 294 len 8192 11/30 Read block 305 len 28672 12/30 Read block 163 len 16384 13/30 Read block 423 len 24576 14/30 Read block 118 len 24576 15/30 Write block 152 len 8192 data: 9 13 16/30 Flush 17/30 Write block 480 len 16384 data: 11 15 13 12 18/30 Read block 579 len 4096 19/30 Write block 528 len 24576 data: 6 6 7 7 9 8 20/30 Write block 169 len 32768 data: 5 4 4 5 10 10 9 10 21/30 Write block 400 len 40960 data: 7 9 13 15 13 10 8 11 12 11 22/30 Flush 23/30 Write block 467 len 28672 data: 7 7 7 8 7 5 5 24/30 Write block 455 len 40960 data: 7 7 5 6 7 6 6 8 10 9 25/30 Write block 152 len 4096 data: 10 26/30 Write block 47 len 16384 data: 12 10 12 17 27/30 Write block 408 len 4096 data: 13 28/30 Read block 472 len 36864 29/30 Read block 517 len 20480 30/30 Write block 36 len 16384 data: 10 10 10 9 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:41.908069934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 108 108 108 189 189 189 T T T 2 040-059 107 107 107 187 187 187 T T T 3 060-079 108 108 108 190 190 190 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 110 110 110 193 193 193 T T T 8 160-179 110 110 110 194 194 194 F F F 9 180-199 110 110 110 192 192 192 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 108 108 108 189 189 189 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 108 108 108 190 190 190 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 107 107 107 187 187 187 F F F 20 400-419 110 110 110 194 194 194 T T T 21 420-439 110 110 110 193 193 193 F F F 22 440-459 110 110 110 193 193 193 T T T 23 460-479 108 108 108 190 190 190 T T T 24 480-499 110 110 110 194 194 194 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 110 110 110 194 194 194 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 110, Max flush: 194 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 111 111 111 195 195 195 F F F 2 040-059 111 111 111 195 195 195 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 F F F 6 120-139 104 104 104 184 184 184 F F F 7 140-159 111 111 111 195 195 195 F F F 8 160-179 110 110 110 194 194 194 F F F 9 180-199 110 110 110 192 192 192 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 108 108 108 189 189 189 F F F 16 320-339 104 104 104 184 184 184 F F F 17 340-359 108 108 108 190 190 190 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 107 107 107 187 187 187 F F F 20 400-419 111 111 111 195 195 195 F F F 21 420-439 110 110 110 193 193 193 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 111 111 111 195 195 195 F F F 24 480-499 110 110 110 194 194 194 F F F 25 500-519 102 102 102 183 183 183 F F F 26 520-539 110 110 110 194 194 194 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 101 101 101 182 182 182 F F F 29 580-599 102 102 102 183 183 183 F F F Max gen: 111, Max flush: 195 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.17583452Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.218145892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.218714436Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.218749626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.218765457Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.218780547Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630} {"msg":"Crucible 188aae24-2b6a-4592-b6b5-ef2aba2318a2 has session id: ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.219234058Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Upstairs opts: Upstairs UUID: 188aae24-2b6a-4592-b6b5-ef2aba2318a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.219281269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Crucible stats registered with UUID: 188aae24-2b6a-4592-b6b5-ef2aba2318a2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.21930607Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.248869656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.248904977Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630} {"msg":"188aae24-2b6a-4592-b6b5-ef2aba2318a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.248936658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.248955288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.248984919Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.24900844Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.24904151Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249059531Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249076851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249093562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249111122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249128483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42030)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249145973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40280)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249336897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47012)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249468331Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249820519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.24984822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249961482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.249997123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.250059165Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.250094995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.250436294Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.250476995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.250739211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251074729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.2511061Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.25112243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.25113548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251149351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251162861Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251331535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251368226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[0]R flush_numbers[0..12]: [191, 195, 195, 190, 185, 164, 184, 195, 194, 192, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251394687Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[0]R generation[0..12]: [109, 111, 111, 108, 105, 94, 104, 111, 110, 110, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251421027Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251447098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[1]R flush_numbers[0..12]: [191, 195, 195, 190, 185, 164, 184, 195, 194, 192, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251474959Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[1]R generation[0..12]: [109, 111, 111, 108, 105, 94, 104, 111, 110, 110, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251501389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.2515295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[2]R flush_numbers[0..12]: [191, 195, 195, 190, 185, 164, 184, 195, 194, 192, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251556281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"[2]R generation[0..12]: [109, 111, 111, 108, 105, 94, 104, 111, 110, 110, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251584281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251624242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251655843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251685644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251715194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251744755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"":"downstairs","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251773996Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251800076Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"0","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251815637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"1","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251829127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"client":"2","":"downstairs","session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251850868Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251863908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"188aae24-2b6a-4592-b6b5-ef2aba2318a2 is now active with session: ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251877038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251889278Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630,"session_id":"ffbd703b-c2a7-4c75-8c40-8ad04a69e5f6"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:46.251901749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 337 len 16384 data: 8 9 8 8 02/30 Read block 30 len 12288 03/30 Write block 500 len 20480 data: 7 9 9 12 11 04/30 Flush 05/30 Flush 06/30 Write block 34 len 36864 data: 10 9 11 11 11 10 9 11 12 07/30 Write block 574 len 8192 data: 11 10 08/30 Read block 493 len 16384 09/30 Read block 327 len 24576 10/30 Read block 539 len 36864 11/30 Read block 0 len 40960 12/30 Read block 140 len 16384 13/30 Flush 14/30 Write block 135 len 40960 data: 6 7 4 5 7 11 14 12 12 12 15/30 Write block 354 len 8192 data: 10 8 16/30 Read block 550 len 32768 17/30 Write block 114 len 32768 data: 5 5 7 7 6 11 12 11 18/30 Read block 335 len 40960 19/30 Write block 66 len 20480 data: 16 15 11 10 9 20/30 Read block 140 len 24576 21/30 Read block 72 len 40960 22/30 Read block 468 len 36864 23/30 Write block 388 len 28672 data: 9 9 10 8 7 7 8 24/30 Write block 598 len 4096 data: 4 25/30 Read block 471 len 4096 26/30 Write block 430 len 4096 data: 12 27/30 Write block 471 len 12288 data: 8 6 6 28/30 Read block 278 len 8192 29/30 Write block 494 len 40960 data: 9 9 8 7 6 7 8 10 10 13 30/30 Write block 552 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:46.270908783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 112 112 112 198 198 198 F F F 3 060-079 108 108 108 190 190 190 T T T 4 080-099 105 105 105 185 185 185 F F F 5 100-119 94 94 94 164 164 164 T T T 6 120-139 104 104 104 184 184 184 T T T 7 140-159 111 111 111 195 195 195 T T T 8 160-179 110 110 110 194 194 194 F F F 9 180-199 110 110 110 192 192 192 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 108 108 108 189 189 189 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 112 112 112 196 196 196 T T T 18 360-379 107 107 107 187 187 187 F F F 19 380-399 107 107 107 187 187 187 T T T 20 400-419 111 111 111 195 195 195 F F F 21 420-439 110 110 110 193 193 193 T T T 22 440-459 111 111 111 195 195 195 F F F 23 460-479 111 111 111 195 195 195 T T T 24 480-499 110 110 110 194 194 194 T T T 25 500-519 112 112 112 196 196 196 T T T 26 520-539 110 110 110 194 194 194 F F F 27 540-559 108 108 108 190 190 190 T T T 28 560-579 112 112 112 198 198 198 F F F 29 580-599 102 102 102 183 183 183 T T T Max gen: 112, Max flush: 198 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 112 112 112 198 198 198 F F F 3 060-079 113 113 113 199 199 199 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 113 113 113 199 199 199 F F F 6 120-139 113 113 113 199 199 199 F F F 7 140-159 113 113 113 199 199 199 F F F 8 160-179 110 110 110 194 194 194 F F F 9 180-199 110 110 110 192 192 192 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 102 102 102 183 183 183 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 108 108 108 189 189 189 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 113 113 113 199 199 199 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 113 113 113 199 199 199 F F F 20 400-419 111 111 111 195 195 195 F F F 21 420-439 113 113 113 199 199 199 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 113 113 113 199 199 199 F F F 24 480-499 113 113 113 199 199 199 F F F 25 500-519 113 113 113 199 199 199 F F F 26 520-539 110 110 110 194 194 194 F F F 27 540-559 113 113 113 199 199 199 F F F 28 560-579 112 112 112 198 198 198 F F F 29 580-599 113 113 113 199 199 199 F F F Max gen: 113, Max flush: 199 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.568924003Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.613528659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.614166124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.614203595Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.614218556Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.614233766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638} {"msg":"Crucible b6742d44-0a9c-4b08-863e-a48e34a7dce6 has session id: 7b52bb36-904b-4b1a-8d63-07e65db49033","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.614644086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Upstairs opts: Upstairs UUID: b6742d44-0a9c-4b08-863e-a48e34a7dce6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.614686847Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Crucible stats registered with UUID: b6742d44-0a9c-4b08-863e-a48e34a7dce6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.614711887Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644412548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644449018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638} {"msg":"b6742d44-0a9c-4b08-863e-a48e34a7dce6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644483849Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644501679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.64453002Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644548101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644571131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644589382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644606442Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644622902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644640453Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644667683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_connection connected from Ok(127.0.0.1:43084)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644700274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_connection connected from Ok(127.0.0.1:57305)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644731975Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_connection connected from Ok(127.0.0.1:54965)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.644764706Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.645510794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.645540744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.645693438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.645723789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.64578235Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.645828821Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646258552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646305823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646443746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646804574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646841395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646867196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646938698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646959438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.646973599Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.64704141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647060901Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[0]R flush_numbers[0..12]: [191, 198, 198, 199, 185, 199, 199, 199, 194, 192, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647075651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[0]R generation[0..12]: [109, 112, 112, 113, 105, 113, 113, 113, 110, 110, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647090361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647104772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[1]R flush_numbers[0..12]: [191, 198, 198, 199, 185, 199, 199, 199, 194, 192, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647118872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[1]R generation[0..12]: [109, 112, 112, 113, 105, 113, 113, 113, 110, 110, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647135993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647150713Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[2]R flush_numbers[0..12]: [191, 198, 198, 199, 185, 199, 199, 199, 194, 192, 190, 183]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647164503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[2]R generation[0..12]: [109, 112, 112, 113, 105, 113, 113, 113, 110, 110, 108, 102]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647186464Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647215214Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647238515Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647255175Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Next flush: 200"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647288486Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647317787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"":"downstairs","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647336907Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647349688Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"0","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647366258Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"1","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647391749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"client":"2","":"downstairs","session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647418729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.6474398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"b6742d44-0a9c-4b08-863e-a48e34a7dce6 is now active with session: 7b52bb36-904b-4b1a-8d63-07e65db49033","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.64745845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647482231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:50.647506481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 487 len 24576 data: 12 10 9 9 9 7 02/30 Read block 247 len 40960 03/30 Read block 209 len 8192 04/30 Read block 568 len 32768 05/30 Read block 579 len 4096 06/30 Flush 07/30 Write block 213 len 36864 data: 6 5 6 8 7 7 8 9 10 08/30 Read block 447 len 12288 09/30 Read block 130 len 12288 10/30 Write block 59 len 16384 data: 10 9 10 9 11/30 Read block 122 len 24576 12/30 Read block 233 len 4096 13/30 Write block 222 len 16384 data: 12 12 11 11 14/30 Read block 121 len 36864 15/30 Write block 463 len 4096 data: 11 16/30 Write block 0 len 40960 data: 3 4 4 6 8 7 6 7 6 8 17/30 Read block 15 len 4096 18/30 Write block 312 len 20480 data: 6 8 9 10 11 19/30 Write block 434 len 8192 data: 8 6 20/30 Write block 473 len 40960 data: 7 3 3 4 5 8 11 12 16 14 21/30 Write block 138 len 20480 data: 6 8 12 15 13 22/30 Write block 531 len 8192 data: 8 10 23/30 Write block 155 len 40960 data: 17 17 15 17 18 15 15 17 13 12 24/30 Read block 500 len 8192 25/30 Write block 340 len 36864 data: 9 8 7 7 10 11 15 16 13 26/30 Flush 27/30 Write block 159 len 40960 data: 19 16 16 18 14 13 12 11 6 5 28/30 Read block 289 len 8192 29/30 Write block 193 len 36864 data: 9 10 11 10 12 10 11 9 9 30/30 Write block 465 len 16384 data: 8 8 8 8 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.666643129Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667325325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667364726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667397147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667430818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667459568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667485999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667511Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66752734Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.6675405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667553081Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667565591Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667577771Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667589882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667602052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667614122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667626412Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667638413Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667650693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667662963Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667675293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667690614Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667703404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667715734Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667727955Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667740065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667752355Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667764606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667776696Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667788886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667800987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667813307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667825517Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667837557Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667849778Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667861878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667874008Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667886118Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667900269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667912669Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667924899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66793698Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.6679493Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66796143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667973671Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667985991Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.667998121Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668010341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668022622Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668034802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668046882Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668059143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668071413Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668083653Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668095814Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668108044Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668121914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668134364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668146785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668158865Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668170995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668183206Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668195356Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668207476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668219657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668231857Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668244117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668256177Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668268558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668280558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668292728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668304839Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668317229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668331059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668343429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66835635Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66836863Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66838088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668393131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668405671Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668418311Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668430931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668443382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668455762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668468203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668480733Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668493213Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668505693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668518114Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668530364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668543964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668556645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668569075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668581465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668593915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668606266Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668618656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668630976Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668643337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668655757Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668668187Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668680608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668692968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668705508Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668717889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668730459Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668742709Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668756599Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66876922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66878149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.66879373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668806011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668818431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668830681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668856442Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668869552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668881842Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668894033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668906173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668918553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668930603Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668942784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668954944Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668967094Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668980925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:50.668993415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1638,"session_id":"7b52bb36-904b-4b1a-8d63-07e65db49033"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 201 201 201 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 114 114 114 201 201 201 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 113 113 113 199 199 199 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 114 114 114 201 201 201 T T T 8 160-179 114 114 114 201 201 201 T T T 9 180-199 110 110 110 192 192 192 T T T 10 200-219 114 114 114 201 201 201 T T T 11 220-239 114 114 114 201 201 201 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 114 114 114 201 201 201 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 113 113 113 199 199 199 F F F 20 400-419 111 111 111 195 195 195 F F F 21 420-439 114 114 114 201 201 201 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 114 114 114 201 201 201 T T T 24 480-499 114 114 114 201 201 201 F F F 25 500-519 113 113 113 199 199 199 F F F 26 520-539 114 114 114 201 201 201 F F F 27 540-559 113 113 113 199 199 199 F F F 28 560-579 112 112 112 198 198 198 F F F 29 580-599 113 113 113 199 199 199 F F F Max gen: 114, Max flush: 201 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 201 201 201 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 114 114 114 201 201 201 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 113 113 113 199 199 199 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 115 115 115 202 202 202 F F F 8 160-179 115 115 115 202 202 202 F F F 9 180-199 115 115 115 202 202 202 F F F 10 200-219 115 115 115 202 202 202 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 F F F 15 300-319 114 114 114 201 201 201 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 113 113 113 199 199 199 F F F 20 400-419 111 111 111 195 195 195 F F F 21 420-439 114 114 114 201 201 201 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 115 115 115 202 202 202 F F F 24 480-499 114 114 114 201 201 201 F F F 25 500-519 113 113 113 199 199 199 F F F 26 520-539 114 114 114 201 201 201 F F F 27 540-559 113 113 113 199 199 199 F F F 28 560-579 112 112 112 198 198 198 F F F 29 580-599 113 113 113 199 199 199 F F F Max gen: 115, Max flush: 202 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.926130157Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.96892809Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.969568925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.969606016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.969621597Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.969636577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646} {"msg":"Crucible 095b9ea6-6d7f-4edd-97d3-b73e56fcd472 has session id: 8f8458d6-98c8-4b25-94d7-97c51c31f33f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.970023756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Upstairs opts: Upstairs UUID: 095b9ea6-6d7f-4edd-97d3-b73e56fcd472, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.970066928Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Crucible stats registered with UUID: 095b9ea6-6d7f-4edd-97d3-b73e56fcd472","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:54.970091518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000572977Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000611198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646} {"msg":"095b9ea6-6d7f-4edd-97d3-b73e56fcd472 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000640758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000684229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000774971Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000826463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000858753Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000889174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000920515Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.000947006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.001053428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.001088069Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46701)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.001107159Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45302)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.00112238Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63038)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.00113823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.001650572Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.001688823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.001836867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.001881668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.00195366Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.002004231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.002350229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.002651656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.002733908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003045126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003073846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003092497Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003119078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003147788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003172789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.00323952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003259841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[0]R flush_numbers[0..12]: [201, 198, 201, 201, 185, 199, 201, 202, 202, 202, 202, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003274421Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[0]R generation[0..12]: [114, 112, 114, 114, 105, 113, 114, 115, 115, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003288611Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003303092Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[1]R flush_numbers[0..12]: [201, 198, 201, 201, 185, 199, 201, 202, 202, 202, 202, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003321572Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[1]R generation[0..12]: [114, 112, 114, 114, 105, 113, 114, 115, 115, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003336333Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003350613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[2]R flush_numbers[0..12]: [201, 198, 201, 201, 185, 199, 201, 202, 202, 202, 202, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003364573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[2]R generation[0..12]: [114, 112, 114, 114, 105, 113, 114, 115, 115, 115, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003378454Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003392584Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003405794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003419325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003432415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003445615Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"":"downstairs","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:11:55.003459106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003484756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"0","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003513497Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"1","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003540068Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"client":"2","":"downstairs","session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003570508Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003589409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"095b9ea6-6d7f-4edd-97d3-b73e56fcd472 is now active with session: 8f8458d6-98c8-4b25-94d7-97c51c31f33f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003603219Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.003615549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646,"session_id":"8f8458d6-98c8-4b25-94d7-97c51c31f33f"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:55.00362808Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 376 len 36864 data: 6 9 10 9 8 8 9 7 9 02/30 Read block 57 len 20480 03/30 Read block 108 len 28672 04/30 Read block 408 len 36864 05/30 Read block 321 len 16384 06/30 Read block 344 len 40960 07/30 Write block 445 len 24576 data: 6 6 5 5 6 7 08/30 Write block 155 len 20480 data: 18 18 16 18 20 09/30 Write block 116 len 8192 data: 8 8 10/30 Read block 147 len 28672 11/30 Write block 186 len 32768 data: 9 9 6 5 4 7 7 10 12/30 Read block 466 len 8192 13/30 Read block 497 len 8192 14/30 Read block 305 len 32768 15/30 Read block 558 len 16384 16/30 Read block 524 len 8192 17/30 Read block 462 len 4096 18/30 Write block 397 len 28672 data: 10 10 8 8 10 14 16 19/30 Read block 324 len 12288 20/30 Read block 183 len 12288 21/30 Read block 54 len 20480 22/30 Read block 360 len 36864 23/30 Write block 290 len 12288 data: 10 12 14 24/30 Read block 333 len 36864 25/30 Read block 94 len 16384 26/30 Read block 466 len 4096 27/30 Write block 575 len 28672 data: 11 10 9 10 9 9 9 28/30 Write block 417 len 40960 data: 8 6 6 7 8 11 11 10 12 12 29/30 Write block 513 len 4096 data: 7 30/30 Write block 161 len 32768 data: 17 19 15 14 13 12 7 6 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:55.023256999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 201 201 201 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 114 114 114 201 201 201 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 113 113 113 199 199 199 T T T 6 120-139 114 114 114 201 201 201 F F F 7 140-159 115 115 115 202 202 202 T T T 8 160-179 115 115 115 202 202 202 T T T 9 180-199 115 115 115 202 202 202 T T T 10 200-219 115 115 115 202 202 202 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 107 107 107 187 187 187 T T T 15 300-319 114 114 114 201 201 201 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 107 107 107 187 187 187 T T T 19 380-399 113 113 113 199 199 199 T T T 20 400-419 111 111 111 195 195 195 T T T 21 420-439 114 114 114 201 201 201 T T T 22 440-459 111 111 111 195 195 195 T T T 23 460-479 115 115 115 202 202 202 F F F 24 480-499 114 114 114 201 201 201 F F F 25 500-519 113 113 113 199 199 199 T T T 26 520-539 114 114 114 201 201 201 F F F 27 540-559 113 113 113 199 199 199 F F F 28 560-579 112 112 112 198 198 198 T T T 29 580-599 113 113 113 199 199 199 T T T Max gen: 115, Max flush: 202 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 201 201 201 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 114 114 114 201 201 201 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 105 105 105 185 185 185 F F F 5 100-119 116 116 116 203 203 203 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 116 116 116 203 203 203 F F F 10 200-219 115 115 115 202 202 202 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 105 105 105 185 185 185 F F F 14 280-299 116 116 116 203 203 203 F F F 15 300-319 114 114 114 201 201 201 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 116 116 116 203 203 203 F F F 19 380-399 116 116 116 203 203 203 F F F 20 400-419 116 116 116 203 203 203 F F F 21 420-439 116 116 116 203 203 203 F F F 22 440-459 116 116 116 203 203 203 F F F 23 460-479 115 115 115 202 202 202 F F F 24 480-499 114 114 114 201 201 201 F F F 25 500-519 116 116 116 203 203 203 F F F 26 520-539 114 114 114 201 201 201 F F F 27 540-559 113 113 113 199 199 199 F F F 28 560-579 116 116 116 203 203 203 F F F 29 580-599 116 116 116 203 203 203 F F F Max gen: 116, Max flush: 203 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.325204093Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:59.36650943Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.366865779Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.39744236Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.397990783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.398031884Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.398047304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.398073325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654} {"msg":"Crucible 62c23992-caed-4677-9c4e-bc8247a728b0 has session id: 4f3af7ce-fbfc-4fd1-b971-4269589ce80c","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.398469565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Upstairs opts: Upstairs UUID: 62c23992-caed-4677-9c4e-bc8247a728b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.398517866Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Crucible stats registered with UUID: 62c23992-caed-4677-9c4e-bc8247a728b0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.398543196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.42796953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428006391Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654} {"msg":"62c23992-caed-4677-9c4e-bc8247a728b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428066992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428095953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428122303Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428137694Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428152514Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428167444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428191915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428217305Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428242236Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428268987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52509)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428479212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56197)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428520783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64786)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.428548804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.429150178Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.429191329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.42922717Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.42925953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.429291151Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.429317632Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.429876605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.429903716Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.429971678Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.43047721Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.4305011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430515111Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430588542Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430608213Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430621823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430673654Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430691805Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[0]R flush_numbers[0..12]: [201, 198, 201, 201, 185, 203, 201, 203, 203, 203, 202, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430710245Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[0]R generation[0..12]: [114, 112, 114, 114, 105, 116, 114, 116, 116, 116, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430724936Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430739236Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[1]R flush_numbers[0..12]: [201, 198, 201, 201, 185, 203, 201, 203, 203, 203, 202, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430753106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[1]R generation[0..12]: [114, 112, 114, 114, 105, 116, 114, 116, 116, 116, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430766846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430780747Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[2]R flush_numbers[0..12]: [201, 198, 201, 201, 185, 203, 201, 203, 203, 203, 202, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430794727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[2]R generation[0..12]: [114, 112, 114, 114, 105, 116, 114, 116, 116, 116, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430808538Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430822458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Max found gen is 117","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430835508Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Generation requested: 118 >= found:117","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430848699Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430861739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"All extents match","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2024-12-09T15:11:59.430882409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"":"downstairs","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.43089568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.43091037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"0","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.43092386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"1","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.43093701Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"client":"2","":"downstairs","session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430950361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430962891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"62c23992-caed-4677-9c4e-bc8247a728b0 is now active with session: 4f3af7ce-fbfc-4fd1-b971-4269589ce80c","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430975591Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430987482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654,"session_id":"4f3af7ce-fbfc-4fd1-b971-4269589ce80c"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:11:59.430999792Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 320 len 36864 data: 10 10 8 10 10 10 11 11 9 02/30 Write block 533 len 32768 data: 9 8 9 10 11 10 10 11 03/30 Write block 574 len 36864 data: 12 12 11 10 11 10 10 10 8 04/30 Read block 462 len 28672 05/30 Read block 567 len 36864 06/30 Write block 44 len 28672 data: 11 12 14 13 11 13 18 07/30 Read block 428 len 32768 08/30 Read block 485 len 32768 09/30 Write block 201 len 12288 data: 10 7 8 10/30 Read block 187 len 24576 11/30 Write block 125 len 12288 data: 10 12 12 12/30 Write block 184 len 28672 data: 8 9 10 10 7 6 5 13/30 Write block 292 len 36864 data: 15 11 11 11 11 14 10 7 6 14/30 Write block 198 len 24576 data: 11 12 10 11 8 9 15/30 Read block 170 len 4096 16/30 Read block 340 len 12288 17/30 Read block 149 len 16384 18/30 Read block 223 len 24576 19/30 Read block 435 len 20480 20/30 Write block 260 len 16384 data: 10 8 7 6 21/30 Flush 22/30 Write block 540 len 36864 data: 12 9 9 11 11 10 8 8 8 23/30 Write block 195 len 16384 data: 12 11 13 12 24/30 Write block 87 len 32768 data: 3 6 6 7 7 8 8 6 25/30 Flush 26/30 Write block 332 len 36864 data: 7 9 10 10 8 9 10 9 10 27/30 Read block 160 len 12288 28/30 Read block 248 len 20480 29/30 Read block 529 len 32768 30/30 Read block 151 len 8192 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:11:59.45139745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 201 201 201 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 118 118 118 205 205 205 F F F 5 100-119 116 116 116 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 118 118 118 205 205 205 F F F 10 200-219 118 118 118 204 204 204 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 118 118 118 204 204 204 F F F 14 280-299 118 118 118 204 204 204 F F F 15 300-319 118 118 118 204 204 204 F F F 16 320-339 118 118 118 204 204 204 T T T 17 340-359 114 114 114 201 201 201 T T T 18 360-379 116 116 116 203 203 203 F F F 19 380-399 116 116 116 203 203 203 F F F 20 400-419 116 116 116 203 203 203 F F F 21 420-439 116 116 116 203 203 203 F F F 22 440-459 116 116 116 203 203 203 F F F 23 460-479 115 115 115 202 202 202 F F F 24 480-499 114 114 114 201 201 201 F F F 25 500-519 116 116 116 203 203 203 F F F 26 520-539 118 118 118 204 204 204 F F F 27 540-559 118 118 118 205 205 205 F F F 28 560-579 118 118 118 204 204 204 F F F 29 580-599 118 118 118 204 204 204 F F F Max gen: 118, Max flush: 205 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 201 201 201 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 118 118 118 205 205 205 F F F 5 100-119 116 116 116 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 118 118 118 205 205 205 F F F 10 200-219 118 118 118 204 204 204 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 110 110 110 192 192 192 F F F 13 260-279 118 118 118 204 204 204 F F F 14 280-299 118 118 118 204 204 204 F F F 15 300-319 118 118 118 204 204 204 F F F 16 320-339 119 119 119 206 206 206 F F F 17 340-359 119 119 119 206 206 206 F F F 18 360-379 116 116 116 203 203 203 F F F 19 380-399 116 116 116 203 203 203 F F F 20 400-419 116 116 116 203 203 203 F F F 21 420-439 116 116 116 203 203 203 F F F 22 440-459 116 116 116 203 203 203 F F F 23 460-479 115 115 115 202 202 202 F F F 24 480-499 114 114 114 201 201 201 F F F 25 500-519 116 116 116 203 203 203 F F F 26 520-539 118 118 118 204 204 204 F F F 27 540-559 118 118 118 205 205 205 F F F 28 560-579 118 118 118 204 204 204 F F F 29 580-599 118 118 118 204 204 204 F F F Max gen: 119, Max flush: 206 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.725080788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.767721817Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.768352682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.768389923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.768404613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.768419254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662} {"msg":"Crucible 63dd0da5-72b0-47ea-b811-ae8bbda3bd63 has session id: be428d07-60b1-45cf-a852-5f2dc6b571cf","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.768866485Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Upstairs opts: Upstairs UUID: 63dd0da5-72b0-47ea-b811-ae8bbda3bd63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.768913946Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Crucible stats registered with UUID: 63dd0da5-72b0-47ea-b811-ae8bbda3bd63","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.768942077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.798801039Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.79883426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662} {"msg":"63dd0da5-72b0-47ea-b811-ae8bbda3bd63 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.798861481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.798878141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.798907982Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.798931493Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.798954623Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.798986814Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799021295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799054776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799092196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799127587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58166)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799161248Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:54679)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799195639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38869)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.79924568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799923566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799965037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.799997778Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.800028329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.800064369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.80008608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.800599912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.800747876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.800789127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801102965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801129895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801145406Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801299289Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.80131893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.80133268Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801403762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801423082Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[0]R flush_numbers[0..12]: [201, 198, 204, 201, 205, 203, 204, 203, 203, 205, 204, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801437562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[0]R generation[0..12]: [114, 112, 118, 114, 118, 116, 118, 116, 116, 118, 118, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801451663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801465833Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[1]R flush_numbers[0..12]: [201, 198, 204, 201, 205, 203, 204, 203, 203, 205, 204, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801483174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[1]R generation[0..12]: [114, 112, 118, 114, 118, 116, 118, 116, 116, 118, 118, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801497564Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801511474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[2]R flush_numbers[0..12]: [201, 198, 204, 201, 205, 203, 204, 203, 203, 205, 204, 201]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801525475Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[2]R generation[0..12]: [114, 112, 118, 114, 118, 116, 118, 116, 116, 118, 118, 114]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801539345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801553245Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801566325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801579876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801593266Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-12-09T15:12:03.801606277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"":"downstairs","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801627947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801640087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"0","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801653407Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"1","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801666528Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"client":"2","":"downstairs","session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801679748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801694079Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"63dd0da5-72b0-47ea-b811-ae8bbda3bd63 is now active with session: be428d07-60b1-45cf-a852-5f2dc6b571cf","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801707469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801719469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662,"session_id":"be428d07-60b1-45cf-a852-5f2dc6b571cf"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:03.801731729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 24576 data: 10 9 8 6 7 7 02/30 Read block 534 len 36864 03/30 Write block 288 len 40960 data: 11 11 11 13 16 12 12 12 12 15 04/30 Write block 117 len 8192 data: 9 7 05/30 Write block 241 len 16384 data: 8 8 7 6 06/30 Write block 295 len 16384 data: 13 13 16 11 07/30 Read block 103 len 28672 08/30 Write block 378 len 8192 data: 11 10 09/30 Read block 578 len 8192 10/30 Write block 219 len 12288 data: 9 10 11 11/30 Read block 524 len 4096 12/30 Write block 502 len 4096 data: 11 13/30 Read block 196 len 8192 14/30 Write block 590 len 32768 data: 7 7 7 6 7 6 6 5 15/30 Write block 501 len 28672 data: 11 12 14 12 10 12 13 16/30 Write block 119 len 28672 data: 12 13 12 11 11 10 11 17/30 Write block 107 len 16384 data: 7 6 7 6 18/30 Write block 7 len 24576 data: 8 7 9 10 7 8 19/30 Write block 332 len 16384 data: 8 10 11 11 20/30 Write block 552 len 36864 data: 12 9 11 9 9 12 10 7 5 21/30 Write block 133 len 24576 data: 6 6 7 8 5 7 22/30 Write block 194 len 36864 data: 11 13 12 14 13 13 11 12 9 23/30 Read block 5 len 40960 24/30 Read block 483 len 4096 25/30 Read block 403 len 36864 26/30 Read block 298 len 28672 27/30 Read block 567 len 16384 28/30 Read block 211 len 12288 29/30 Write block 93 len 4096 data: 9 30/30 Write block 219 len 8192 data: 10 11 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:03.818584702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 201 201 201 T T T 1 020-039 112 112 112 198 198 198 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 118 118 118 205 205 205 T T T 5 100-119 116 116 116 203 203 203 T T T 6 120-139 118 118 118 204 204 204 T T T 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 118 118 118 205 205 205 T T T 10 200-219 118 118 118 204 204 204 T T T 11 220-239 114 114 114 201 201 201 T T T 12 240-259 110 110 110 192 192 192 T T T 13 260-279 118 118 118 204 204 204 F F F 14 280-299 118 118 118 204 204 204 T T T 15 300-319 118 118 118 204 204 204 T T T 16 320-339 119 119 119 206 206 206 T T T 17 340-359 119 119 119 206 206 206 F F F 18 360-379 116 116 116 203 203 203 T T T 19 380-399 116 116 116 203 203 203 F F F 20 400-419 116 116 116 203 203 203 F F F 21 420-439 116 116 116 203 203 203 F F F 22 440-459 116 116 116 203 203 203 F F F 23 460-479 115 115 115 202 202 202 F F F 24 480-499 114 114 114 201 201 201 F F F 25 500-519 116 116 116 203 203 203 T T T 26 520-539 118 118 118 204 204 204 F F F 27 540-559 118 118 118 205 205 205 T T T 28 560-579 118 118 118 204 204 204 T T T 29 580-599 118 118 118 204 204 204 T T T Max gen: 119, Max flush: 206 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 112 112 112 198 198 198 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 114 114 114 201 201 201 F F F 4 080-099 120 120 120 207 207 207 F F F 5 100-119 120 120 120 207 207 207 F F F 6 120-139 120 120 120 207 207 207 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 120 120 120 207 207 207 F F F 10 200-219 120 120 120 207 207 207 F F F 11 220-239 120 120 120 207 207 207 F F F 12 240-259 120 120 120 207 207 207 F F F 13 260-279 118 118 118 204 204 204 F F F 14 280-299 120 120 120 207 207 207 F F F 15 300-319 120 120 120 207 207 207 F F F 16 320-339 120 120 120 207 207 207 F F F 17 340-359 119 119 119 206 206 206 F F F 18 360-379 120 120 120 207 207 207 F F F 19 380-399 116 116 116 203 203 203 F F F 20 400-419 116 116 116 203 203 203 F F F 21 420-439 116 116 116 203 203 203 F F F 22 440-459 116 116 116 203 203 203 F F F 23 460-479 115 115 115 202 202 202 F F F 24 480-499 114 114 114 201 201 201 F F F 25 500-519 120 120 120 207 207 207 F F F 26 520-539 118 118 118 204 204 204 F F F 27 540-559 120 120 120 207 207 207 F F F 28 560-579 120 120 120 207 207 207 F F F 29 580-599 120 120 120 207 207 207 F F F Max gen: 120, Max flush: 207 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.13785506Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:08.180364087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.180727515Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.212531256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.213246173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.213282814Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.213297594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.213312034Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670} {"msg":"Crucible dafa2839-1aef-4830-a287-6bf1c3b0b37d has session id: 6c20e438-ba0d-41bc-8846-7e76350136e5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.213751005Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Upstairs opts: Upstairs UUID: dafa2839-1aef-4830-a287-6bf1c3b0b37d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.213786075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Crucible stats registered with UUID: dafa2839-1aef-4830-a287-6bf1c3b0b37d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.213802126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244236653Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244268384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670} {"msg":"dafa2839-1aef-4830-a287-6bf1c3b0b37d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244344646Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244368377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244399037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244419058Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244450679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244554101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244575682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244607482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244643383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244674624Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46026)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244708035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60119)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244756656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41308)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.244792467Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.24535668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.245392321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.245409151Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.245428662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.245669618Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.245698458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246224061Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246267322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246303403Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246798435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246828235Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246843166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246855846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246869586Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246883627Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246896047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246909487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[0]R flush_numbers[0..12]: [207, 198, 204, 201, 207, 207, 207, 203, 203, 207, 207, 207]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246927708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[0]R generation[0..12]: [120, 112, 118, 114, 120, 120, 120, 116, 116, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246942628Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246957988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[1]R flush_numbers[0..12]: [207, 198, 204, 201, 207, 207, 207, 203, 203, 207, 207, 207]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246972159Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[1]R generation[0..12]: [120, 112, 118, 114, 120, 120, 120, 116, 116, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.246986089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247000089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[2]R flush_numbers[0..12]: [207, 198, 204, 201, 207, 207, 207, 203, 203, 207, 207, 207]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.24701409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[2]R generation[0..12]: [120, 112, 118, 114, 120, 120, 120, 116, 116, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.24702802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.24704209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Max found gen is 121","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247055141Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Generation requested: 122 >= found:121","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:12:08.247068401Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247087142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247100122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"":"downstairs","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247113422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247127772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"0","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247141663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"1","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247155023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"client":"2","":"downstairs","session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247168464Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247181064Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"dafa2839-1aef-4830-a287-6bf1c3b0b37d is now active with session: 6c20e438-ba0d-41bc-8846-7e76350136e5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247193874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247205984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670,"session_id":"6c20e438-ba0d-41bc-8846-7e76350136e5"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:08.247218385Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 545 len 28672 data: 11 9 9 9 6 9 8 02/30 Write block 290 len 36864 data: 12 14 17 13 13 14 14 17 12 03/30 Write block 456 len 40960 data: 8 6 7 8 7 7 9 12 10 9 04/30 Write block 492 len 40960 data: 8 7 10 10 9 8 7 8 9 12 05/30 Write block 238 len 36864 data: 10 9 7 9 9 8 7 7 6 06/30 Write block 321 len 20480 data: 11 9 11 11 11 07/30 Write block 33 len 24576 data: 10 11 10 12 12 12 08/30 Write block 20 len 36864 data: 11 11 12 11 11 14 13 9 9 09/30 Read block 431 len 4096 10/30 Read block 519 len 8192 11/30 Write block 374 len 40960 data: 5 4 7 10 12 11 9 9 10 8 12/30 Read block 190 len 28672 13/30 Read block 528 len 28672 14/30 Flush 15/30 Write block 523 len 32768 data: 7 6 6 4 5 7 7 8 16/30 Write block 310 len 36864 data: 8 8 7 9 10 11 12 11 10 17/30 Read block 257 len 16384 18/30 Read block 268 len 4096 19/30 Write block 58 len 40960 data: 11 11 10 11 10 10 12 15 17 16 20/30 Write block 454 len 40960 data: 9 8 9 7 8 9 8 8 10 13 21/30 Write block 68 len 20480 data: 12 11 10 7 6 22/30 Write block 189 len 8192 data: 7 6 23/30 Write block 268 len 32768 data: 11 14 13 11 12 12 9 9 24/30 Write block 413 len 40960 data: 11 10 9 8 9 7 7 8 9 12 25/30 Flush 26/30 Write block 434 len 12288 data: 9 7 6 27/30 Write block 326 len 32768 data: 12 12 10 9 8 7 9 11 28/30 Write block 122 len 24576 data: 12 12 11 12 13 13 29/30 Read block 268 len 20480 30/30 Read block 213 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:08.266970937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 122 122 122 209 209 209 F F F 3 060-079 122 122 122 209 209 209 F F F 4 080-099 120 120 120 207 207 207 F F F 5 100-119 120 120 120 207 207 207 F F F 6 120-139 120 120 120 207 207 207 T T T 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 120 120 120 207 207 207 F F F 11 220-239 122 122 122 208 208 208 F F F 12 240-259 122 122 122 208 208 208 F F F 13 260-279 122 122 122 209 209 209 F F F 14 280-299 122 122 122 208 208 208 F F F 15 300-319 122 122 122 209 209 209 F F F 16 320-339 122 122 122 208 208 208 T T T 17 340-359 119 119 119 206 206 206 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 209 209 209 F F F 21 420-439 122 122 122 209 209 209 T T T 22 440-459 122 122 122 209 209 209 F F F 23 460-479 122 122 122 209 209 209 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 122 122 122 208 208 208 F F F 26 520-539 122 122 122 209 209 209 F F F 27 540-559 122 122 122 208 208 208 F F F 28 560-579 120 120 120 207 207 207 F F F 29 580-599 120 120 120 207 207 207 F F F Max gen: 122, Max flush: 209 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 122 122 122 209 209 209 F F F 3 060-079 122 122 122 209 209 209 F F F 4 080-099 120 120 120 207 207 207 F F F 5 100-119 120 120 120 207 207 207 F F F 6 120-139 123 123 123 210 210 210 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 120 120 120 207 207 207 F F F 11 220-239 122 122 122 208 208 208 F F F 12 240-259 122 122 122 208 208 208 F F F 13 260-279 122 122 122 209 209 209 F F F 14 280-299 122 122 122 208 208 208 F F F 15 300-319 122 122 122 209 209 209 F F F 16 320-339 123 123 123 210 210 210 F F F 17 340-359 119 119 119 206 206 206 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 209 209 209 F F F 21 420-439 123 123 123 210 210 210 F F F 22 440-459 122 122 122 209 209 209 F F F 23 460-479 122 122 122 209 209 209 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 122 122 122 208 208 208 F F F 26 520-539 122 122 122 209 209 209 F F F 27 540-559 122 122 122 208 208 208 F F F 28 560-579 120 120 120 207 207 207 F F F 29 580-599 120 120 120 207 207 207 F F F Max gen: 123, Max flush: 210 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.552645932Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.595322202Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.595955887Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.596003518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.596020629Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.596035389Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678} {"msg":"Crucible 32a8aed3-062a-46e0-9755-44a3d60d79bb has session id: 4ac60664-1646-4e67-a477-c5b93b6f8c7b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.59649846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Upstairs opts: Upstairs UUID: 32a8aed3-062a-46e0-9755-44a3d60d79bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.596544871Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Crucible stats registered with UUID: 32a8aed3-062a-46e0-9755-44a3d60d79bb","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.596569722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62620392Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626248371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678} {"msg":"32a8aed3-062a-46e0-9755-44a3d60d79bb active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626322983Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626361414Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626394445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626462106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626517058Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626584449Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62661037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62662589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626652191Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626681052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61634)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626711012Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40072)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626742853Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50034)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.626774404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62745366Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.627495411Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.627529772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.627557353Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.627592373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.627611074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628123736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62827527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628315911Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628753031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628785102Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628800602Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628813432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628827143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628840743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628853384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628866814Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[0]R flush_numbers[0..12]: [207, 208, 209, 209, 207, 207, 210, 203, 203, 209, 207, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628881744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[0]R generation[0..12]: [120, 122, 122, 122, 120, 120, 123, 116, 116, 122, 120, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628895985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628914685Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[1]R flush_numbers[0..12]: [207, 208, 209, 209, 207, 207, 210, 203, 203, 209, 207, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628929895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[1]R generation[0..12]: [120, 122, 122, 122, 120, 120, 123, 116, 116, 122, 120, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628944116Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628958406Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[2]R flush_numbers[0..12]: [207, 208, 209, 209, 207, 207, 210, 203, 203, 209, 207, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628972416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[2]R generation[0..12]: [120, 122, 122, 122, 120, 120, 123, 116, 116, 122, 120, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.628986307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629000307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629014157Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629027618Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629040858Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629062478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"":"downstairs","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629075799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629088099Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"0","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62910173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"1","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62911788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"client":"2","":"downstairs","session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.62913144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629144051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"32a8aed3-062a-46e0-9755-44a3d60d79bb is now active with session: 4ac60664-1646-4e67-a477-c5b93b6f8c7b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629157101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629169631Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678,"session_id":"4ac60664-1646-4e67-a477-c5b93b6f8c7b"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:12.629182271Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 28672 data: 14 13 11 10 11 10 11 02/30 Write block 247 len 8192 data: 5 4 03/30 Write block 349 len 24576 data: 14 12 11 12 11 12 04/30 Read block 89 len 28672 05/30 Write block 24 len 32768 data: 12 15 14 10 10 8 11 12 06/30 Write block 104 len 12288 data: 11 10 8 07/30 Read block 589 len 16384 08/30 Read block 143 len 8192 09/30 Read block 577 len 20480 10/30 Write block 415 len 20480 data: 10 9 10 8 8 11/30 Read block 331 len 12288 12/30 Read block 297 len 28672 13/30 Read block 496 len 16384 14/30 Read block 411 len 24576 15/30 Read block 352 len 4096 16/30 Write block 93 len 24576 data: 10 7 8 7 7 10 17/30 Write block 27 len 4096 data: 11 18/30 Read block 461 len 12288 19/30 Read block 16 len 32768 20/30 Read block 381 len 24576 21/30 Read block 310 len 32768 22/30 Write block 535 len 32768 data: 10 11 12 11 11 13 10 10 23/30 Read block 501 len 28672 24/30 Read block 126 len 20480 25/30 Write block 571 len 40960 data: 11 12 11 13 13 12 11 12 11 11 26/30 Read block 93 len 36864 27/30 Read block 464 len 4096 28/30 Read block 406 len 28672 29/30 Read block 557 len 36864 30/30 Write block 560 len 24576 data: 6 5 7 8 6 6 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:12.649472876Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 122 122 122 208 208 208 T T T 2 040-059 122 122 122 209 209 209 F F F 3 060-079 122 122 122 209 209 209 F F F 4 080-099 120 120 120 207 207 207 T T T 5 100-119 120 120 120 207 207 207 T T T 6 120-139 123 123 123 210 210 210 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 120 120 120 207 207 207 F F F 11 220-239 122 122 122 208 208 208 F F F 12 240-259 122 122 122 208 208 208 T T T 13 260-279 122 122 122 209 209 209 F F F 14 280-299 122 122 122 208 208 208 F F F 15 300-319 122 122 122 209 209 209 F F F 16 320-339 123 123 123 210 210 210 F F F 17 340-359 119 119 119 206 206 206 T T T 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 209 209 209 T T T 21 420-439 123 123 123 210 210 210 F F F 22 440-459 122 122 122 209 209 209 F F F 23 460-479 122 122 122 209 209 209 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 122 122 122 208 208 208 F F F 26 520-539 122 122 122 209 209 209 T T T 27 540-559 122 122 122 208 208 208 T T T 28 560-579 120 120 120 207 207 207 T T T 29 580-599 120 120 120 207 207 207 T T T Max gen: 123, Max flush: 210 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 124 124 124 211 211 211 F F F 2 040-059 122 122 122 209 209 209 F F F 3 060-079 122 122 122 209 209 209 F F F 4 080-099 124 124 124 211 211 211 F F F 5 100-119 124 124 124 211 211 211 F F F 6 120-139 123 123 123 210 210 210 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 120 120 120 207 207 207 F F F 11 220-239 122 122 122 208 208 208 F F F 12 240-259 124 124 124 211 211 211 F F F 13 260-279 122 122 122 209 209 209 F F F 14 280-299 122 122 122 208 208 208 F F F 15 300-319 122 122 122 209 209 209 F F F 16 320-339 123 123 123 210 210 210 F F F 17 340-359 124 124 124 211 211 211 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 124 124 124 211 211 211 F F F 21 420-439 123 123 123 210 210 210 F F F 22 440-459 122 122 122 209 209 209 F F F 23 460-479 122 122 122 209 209 209 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 122 122 122 208 208 208 F F F 26 520-539 124 124 124 211 211 211 F F F 27 540-559 124 124 124 211 211 211 F F F 28 560-579 124 124 124 211 211 211 F F F 29 580-599 124 124 124 211 211 211 F F F Max gen: 124, Max flush: 211 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.940847137Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.983580379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.984279665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.984315166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.984330186Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.984344787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686} {"msg":"Crucible 66a83430-cbc4-411e-b497-1697b7b806ff has session id: 1b316405-2807-4c74-b0bb-388a4b375994","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.984724966Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Upstairs opts: Upstairs UUID: 66a83430-cbc4-411e-b497-1697b7b806ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.984769447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Crucible stats registered with UUID: 66a83430-cbc4-411e-b497-1697b7b806ff","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:16.984794417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015499802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015541383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686} {"msg":"66a83430-cbc4-411e-b497-1697b7b806ff active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015566993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015585373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015616454Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015634345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015651495Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015668426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015685476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015701996Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015719257Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015736057Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_connection connected from Ok(127.0.0.1:33824)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.015993103Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_connection connected from Ok(127.0.0.1:57084)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.016200798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_connection connected from Ok(127.0.0.1:56284)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.01625804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.016768272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.016798722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.016819933Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.016835994Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.016850274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.016866194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.017477249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.017508019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.01752486Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018065383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018091733Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018106664Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018164355Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018194476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018210746Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018223596Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018237297Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[0]R flush_numbers[0..12]: [207, 211, 209, 209, 211, 211, 210, 203, 203, 209, 207, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018252267Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[0]R generation[0..12]: [120, 124, 122, 122, 124, 124, 123, 116, 116, 122, 120, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018270768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018285548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[1]R flush_numbers[0..12]: [207, 211, 209, 209, 211, 211, 210, 203, 203, 209, 207, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018299549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[1]R generation[0..12]: [120, 124, 122, 122, 124, 124, 123, 116, 116, 122, 120, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018313439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018327629Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[2]R flush_numbers[0..12]: [207, 211, 209, 209, 211, 211, 210, 203, 203, 209, 207, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018341859Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[2]R generation[0..12]: [120, 124, 122, 122, 124, 124, 123, 116, 116, 122, 120, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.01835584Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.0183702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Max found gen is 125","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.0183837Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Generation requested: 126 >= found:125","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018397681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018411071Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018424311Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"":"downstairs","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018505463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018524324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"0","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018540744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"1","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018554915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"client":"2","":"downstairs","session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018568715Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018581415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"66a83430-cbc4-411e-b497-1697b7b806ff is now active with session: 1b316405-2807-4c74-b0bb-388a4b375994","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018594516Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018606716Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686,"session_id":"1b316405-2807-4c74-b0bb-388a4b375994"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:17.018619086Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 58 len 36864 data: 12 12 11 12 11 11 13 16 18 02/30 Write block 306 len 20480 data: 11 10 9 7 9 03/30 Read block 122 len 28672 04/30 Write block 205 len 40960 data: 7 7 8 8 7 5 6 6 7 6 05/30 Read block 270 len 36864 06/30 Read block 255 len 36864 07/30 Read block 30 len 12288 08/30 Write block 267 len 36864 data: 12 12 15 14 12 13 13 10 10 09/30 Read block 465 len 40960 10/30 Flush 11/30 Write block 586 len 32768 data: 8 9 7 9 8 8 8 7 12/30 Write block 283 len 40960 data: 7 9 10 13 14 12 12 13 15 18 13/30 Write block 452 len 40960 data: 8 9 10 9 10 8 9 10 9 9 14/30 Read block 280 len 36864 15/30 Read block 186 len 24576 16/30 Flush 17/30 Flush 18/30 Write block 335 len 4096 data: 12 19/30 Write block 392 len 28672 data: 8 8 9 9 10 11 11 20/30 Write block 540 len 20480 data: 14 11 11 12 12 21/30 Flush 22/30 Write block 427 len 8192 data: 12 13 23/30 Write block 551 len 40960 data: 9 13 10 12 10 10 13 11 8 7 24/30 Read block 420 len 40960 25/30 Flush 26/30 Write block 509 len 20480 data: 9 11 10 8 8 27/30 Write block 332 len 4096 data: 10 28/30 Write block 208 len 24576 data: 9 8 6 7 7 8 29/30 Read block 430 len 20480 30/30 Read block 155 len 28672 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:17.039145047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 124 124 124 211 211 211 F F F 2 040-059 126 126 126 212 212 212 F F F 3 060-079 126 126 126 212 212 212 F F F 4 080-099 124 124 124 211 211 211 F F F 5 100-119 124 124 124 211 211 211 F F F 6 120-139 123 123 123 210 210 210 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 126 126 126 212 212 212 T T T 11 220-239 122 122 122 208 208 208 F F F 12 240-259 124 124 124 211 211 211 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 126 126 126 215 215 215 T T T 17 340-359 124 124 124 211 211 211 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 126 126 126 215 215 215 F F F 20 400-419 124 124 124 211 211 211 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 122 122 122 208 208 208 T T T 26 520-539 124 124 124 211 211 211 F F F 27 540-559 126 126 126 216 216 216 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 126, Max flush: 216 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 124 124 124 211 211 211 F F F 2 040-059 126 126 126 212 212 212 F F F 3 060-079 126 126 126 212 212 212 F F F 4 080-099 124 124 124 211 211 211 F F F 5 100-119 124 124 124 211 211 211 F F F 6 120-139 123 123 123 210 210 210 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 127 127 127 217 217 217 F F F 11 220-239 122 122 122 208 208 208 F F F 12 240-259 124 124 124 211 211 211 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 127 127 127 217 217 217 F F F 17 340-359 124 124 124 211 211 211 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 126 126 126 215 215 215 F F F 20 400-419 124 124 124 211 211 211 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 127 127 127 217 217 217 F F F 26 520-539 124 124 124 211 211 211 F F F 27 540-559 126 126 126 216 216 216 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 127, Max flush: 217 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.292590362Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:21.333909789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.334285518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.36490875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.365463533Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.365497654Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.365511505Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.365526265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694} {"msg":"Crucible a8216e1d-5294-41c4-9a2b-5f237be6fe66 has session id: c9b7c057-5feb-433a-b899-0eb215908759","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.365940755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Upstairs opts: Upstairs UUID: a8216e1d-5294-41c4-9a2b-5f237be6fe66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.365975326Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Crucible stats registered with UUID: a8216e1d-5294-41c4-9a2b-5f237be6fe66","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.365991826Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395357008Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395387979Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694} {"msg":"a8216e1d-5294-41c4-9a2b-5f237be6fe66 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.39545915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395482161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395515762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395543852Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395577613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395656055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395708436Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395742487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.3958539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395896841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_connection connected from Ok(127.0.0.1:47989)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395930552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_connection connected from Ok(127.0.0.1:50771)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395965542Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_connection connected from Ok(127.0.0.1:49284)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.395996823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.396486985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.396528506Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.396560447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.396591828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.396628378Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.396655149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.397174002Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.397412957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.39753102Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.397631432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.397661663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.397678054Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.397909899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.39793256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.3979472Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.397984491Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398001571Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[0]R flush_numbers[0..12]: [207, 211, 212, 212, 211, 211, 210, 203, 203, 209, 217, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398020142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[0]R generation[0..12]: [120, 124, 126, 126, 124, 124, 123, 116, 116, 122, 127, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398034222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398051903Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[1]R flush_numbers[0..12]: [207, 211, 212, 212, 211, 211, 210, 203, 203, 209, 217, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398080443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[1]R generation[0..12]: [120, 124, 126, 126, 124, 124, 123, 116, 116, 122, 127, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398095604Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398109834Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[2]R flush_numbers[0..12]: [207, 211, 212, 212, 211, 211, 210, 203, 203, 209, 217, 208]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398123724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[2]R generation[0..12]: [120, 124, 126, 126, 124, 124, 123, 116, 116, 122, 127, 122]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398137545Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398151525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398164575Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398177806Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:12:21.398190766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398211926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"":"downstairs","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398225267Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398237257Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"0","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398250527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"1","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398263757Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"client":"2","":"downstairs","session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398279668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398292358Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"a8216e1d-5294-41c4-9a2b-5f237be6fe66 is now active with session: c9b7c057-5feb-433a-b899-0eb215908759","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398305198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398323059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694,"session_id":"c9b7c057-5feb-433a-b899-0eb215908759"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:21.398342479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 115 len 24576 data: 6 9 10 8 13 14 02/30 Write block 89 len 32768 data: 7 8 8 9 11 8 9 8 03/30 Read block 216 len 40960 04/30 Read block 189 len 40960 05/30 Read block 201 len 32768 06/30 Read block 4 len 4096 07/30 Write block 369 len 28672 data: 11 7 6 6 5 6 5 08/30 Write block 224 len 24576 data: 12 12 9 9 7 8 09/30 Read block 448 len 4096 10/30 Write block 57 len 24576 data: 12 13 13 12 13 12 11/30 Write block 89 len 40960 data: 8 9 9 10 12 9 10 9 8 11 12/30 Write block 596 len 12288 data: 7 6 5 13/30 Write block 99 len 12288 data: 13 15 14 14/30 Read block 346 len 20480 15/30 Write block 26 len 4096 data: 15 16/30 Write block 262 len 8192 data: 8 7 17/30 Write block 235 len 16384 data: 9 10 10 11 18/30 Flush 19/30 Write block 355 len 36864 data: 9 9 8 7 9 10 10 10 11 20/30 Read block 582 len 24576 21/30 Read block 579 len 4096 22/30 Read block 149 len 28672 23/30 Read block 391 len 24576 24/30 Read block 528 len 24576 25/30 Write block 537 len 32768 data: 13 12 12 15 12 12 13 13 26/30 Read block 116 len 32768 27/30 Read block 589 len 16384 28/30 Read block 198 len 40960 29/30 Read block 547 len 40960 30/30 Read block 399 len 28672 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:21.417994749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 128 128 128 218 218 218 F F F 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 128 128 128 218 218 218 F F F 5 100-119 128 128 128 218 218 218 F F F 6 120-139 128 128 128 218 218 218 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 127 127 127 217 217 217 F F F 11 220-239 128 128 128 218 218 218 F F F 12 240-259 124 124 124 211 211 211 F F F 13 260-279 128 128 128 218 218 218 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 127 127 127 217 217 217 F F F 17 340-359 124 124 124 211 211 211 T T T 18 360-379 128 128 128 218 218 218 T T T 19 380-399 126 126 126 215 215 215 F F F 20 400-419 124 124 124 211 211 211 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 127 127 127 217 217 217 F F F 26 520-539 124 124 124 211 211 211 T T T 27 540-559 126 126 126 216 216 216 T T T 28 560-579 126 126 126 216 216 216 F F F 29 580-599 128 128 128 218 218 218 F F F Max gen: 128, Max flush: 218 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 207 207 207 F F F 1 020-039 128 128 128 218 218 218 F F F 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 128 128 128 218 218 218 F F F 5 100-119 128 128 128 218 218 218 F F F 6 120-139 128 128 128 218 218 218 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 116 116 116 203 203 203 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 127 127 127 217 217 217 F F F 11 220-239 128 128 128 218 218 218 F F F 12 240-259 124 124 124 211 211 211 F F F 13 260-279 128 128 128 218 218 218 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 127 127 127 217 217 217 F F F 17 340-359 129 129 129 219 219 219 F F F 18 360-379 129 129 129 219 219 219 F F F 19 380-399 126 126 126 215 215 215 F F F 20 400-419 124 124 124 211 211 211 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 127 127 127 217 217 217 F F F 26 520-539 129 129 129 219 219 219 F F F 27 540-559 129 129 129 219 219 219 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 128 128 128 218 218 218 F F F Max gen: 129, Max flush: 219 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.702347053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:25.744706435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.745084154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.776366261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.776906234Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.776939905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.776954115Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.776968745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702} {"msg":"Crucible e893b596-15ad-4a60-9513-887fa2f2a9ba has session id: 6a1cb346-8f56-4df6-a5d8-48a2fb27b749","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.777381546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Upstairs opts: Upstairs UUID: e893b596-15ad-4a60-9513-887fa2f2a9ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.777420856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Crucible stats registered with UUID: e893b596-15ad-4a60-9513-887fa2f2a9ba","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.777437897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.806944042Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.806970683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702} {"msg":"e893b596-15ad-4a60-9513-887fa2f2a9ba active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807048275Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807066475Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807094686Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807111346Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807125757Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807139647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807153617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807171078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807195838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807222469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_connection connected from Ok(127.0.0.1:47360)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807526766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_connection connected from Ok(127.0.0.1:39846)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807576527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_connection connected from Ok(127.0.0.1:48526)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.807607108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808168471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808197422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808212383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808226393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808240303Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808258324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808872578Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808901449Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.808917229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809422231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809451552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809475193Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809497623Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809512743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809526364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809538584Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809552104Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[0]R flush_numbers[0..12]: [207, 218, 218, 218, 218, 218, 218, 203, 203, 209, 217, 218]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809565915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[0]R generation[0..12]: [120, 128, 128, 128, 128, 128, 128, 116, 116, 122, 127, 128]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809580055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809594296Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[1]R flush_numbers[0..12]: [207, 218, 218, 218, 218, 218, 218, 203, 203, 209, 217, 218]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809612186Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[1]R generation[0..12]: [120, 128, 128, 128, 128, 128, 128, 116, 116, 122, 127, 128]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809627036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809641447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[2]R flush_numbers[0..12]: [207, 218, 218, 218, 218, 218, 218, 203, 203, 209, 217, 218]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809655417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[2]R generation[0..12]: [120, 128, 128, 128, 128, 128, 128, 116, 116, 122, 127, 128]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809669097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809683028Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809706068Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809730819Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809749899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.80976994Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"":"downstairs","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.8097939Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809807911Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"0","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809822431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"1","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809846342Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"client":"2","":"downstairs","session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"All required reconciliation work is completed","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809871792Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809886343Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"e893b596-15ad-4a60-9513-887fa2f2a9ba is now active with session: 6a1cb346-8f56-4df6-a5d8-48a2fb27b749","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809899703Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809911823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702,"session_id":"6a1cb346-8f56-4df6-a5d8-48a2fb27b749"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:25.809924363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 318 len 16384 data: 11 12 11 12 02/30 Write block 164 len 12288 data: 15 14 13 03/30 Read block 544 len 36864 04/30 Read block 377 len 36864 05/30 Write block 269 len 8192 data: 16 15 06/30 Write block 124 len 8192 data: 12 13 07/30 Flush 08/30 Write block 89 len 20480 data: 9 10 10 11 13 09/30 Read block 61 len 20480 10/30 Write block 238 len 20480 data: 12 10 8 10 10 11/30 Read block 461 len 4096 12/30 Read block 401 len 12288 13/30 Write block 153 len 40960 data: 14 16 19 19 17 19 21 17 18 20 14/30 Write block 121 len 24576 data: 13 13 13 13 14 14 15/30 Write block 406 len 36864 data: 9 12 14 12 10 10 12 12 11 16/30 Read block 445 len 20480 17/30 Write block 505 len 12288 data: 11 13 14 18/30 Read block 553 len 16384 19/30 Read block 111 len 40960 20/30 Read block 204 len 16384 21/30 Read block 540 len 28672 22/30 Write block 9 len 12288 data: 10 11 8 23/30 Flush 24/30 Write block 81 len 16384 data: 8 7 7 7 25/30 Flush 26/30 Write block 88 len 36864 data: 7 10 11 11 12 14 10 11 10 27/30 Write block 369 len 20480 data: 12 8 7 7 6 28/30 Write block 253 len 12288 data: 10 11 11 29/30 Read block 242 len 36864 30/30 Read block 127 len 16384 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:25.828641691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 221 221 221 F F F 1 020-039 128 128 128 218 218 218 F F F 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 130 130 130 222 222 222 T T T 5 100-119 128 128 128 218 218 218 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 127 127 127 217 217 217 F F F 11 220-239 130 130 130 221 221 221 F F F 12 240-259 130 130 130 221 221 221 T T T 13 260-279 130 130 130 220 220 220 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 130 130 130 220 220 220 F F F 16 320-339 130 130 130 220 220 220 F F F 17 340-359 129 129 129 219 219 219 F F F 18 360-379 129 129 129 219 219 219 T T T 19 380-399 126 126 126 215 215 215 F F F 20 400-419 130 130 130 221 221 221 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 130 130 130 221 221 221 F F F 26 520-539 129 129 129 219 219 219 F F F 27 540-559 129 129 129 219 219 219 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 128 128 128 218 218 218 F F F Max gen: 130, Max flush: 222 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 221 221 221 F F F 1 020-039 128 128 128 218 218 218 F F F 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 131 131 131 223 223 223 F F F 5 100-119 128 128 128 218 218 218 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 127 127 127 217 217 217 F F F 11 220-239 130 130 130 221 221 221 F F F 12 240-259 131 131 131 223 223 223 F F F 13 260-279 130 130 130 220 220 220 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 130 130 130 220 220 220 F F F 16 320-339 130 130 130 220 220 220 F F F 17 340-359 129 129 129 219 219 219 F F F 18 360-379 131 131 131 223 223 223 F F F 19 380-399 126 126 126 215 215 215 F F F 20 400-419 130 130 130 221 221 221 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 130 130 130 221 221 221 F F F 26 520-539 129 129 129 219 219 219 F F F 27 540-559 129 129 129 219 219 219 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 128 128 128 218 218 218 F F F Max gen: 131, Max flush: 223 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.104538052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.148045892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.148670077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.148726158Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.148751459Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.1487787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710} {"msg":"Crucible 35f06a25-ba31-4774-8160-540ba4fce409 has session id: 076870a5-f51e-4d96-9bbb-b0ced4a91763","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.149162559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Upstairs opts: Upstairs UUID: 35f06a25-ba31-4774-8160-540ba4fce409, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.14920785Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Crucible stats registered with UUID: 35f06a25-ba31-4774-8160-540ba4fce409","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.149233731Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179597886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179629837Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710} {"msg":"35f06a25-ba31-4774-8160-540ba4fce409 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179690128Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179718189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.17974519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.17976023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179774541Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179788181Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179805471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179819672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179833652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179847462Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_connection connected from Ok(127.0.0.1:47354)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.179985166Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_connection connected from Ok(127.0.0.1:41649)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.180028117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_connection connected from Ok(127.0.0.1:47263)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.180120879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.180849446Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.180918368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.1810045Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.181041621Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.181073642Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.181104152Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.181502322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.181616075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.181668906Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182129957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182159298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182174608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.18224738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.18226793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182281841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182294201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182307511Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[0]R flush_numbers[0..12]: [221, 218, 218, 218, 223, 218, 221, 221, 221, 209, 217, 221]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182321431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[0]R generation[0..12]: [130, 128, 128, 128, 131, 128, 130, 130, 130, 122, 127, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182335662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182350142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[1]R flush_numbers[0..12]: [221, 218, 218, 218, 223, 218, 221, 221, 221, 209, 217, 221]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182364123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[1]R generation[0..12]: [130, 128, 128, 128, 131, 128, 130, 130, 130, 122, 127, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182377923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182392053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[2]R flush_numbers[0..12]: [221, 218, 218, 218, 223, 218, 221, 221, 221, 209, 217, 221]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182409364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"[2]R generation[0..12]: [130, 128, 128, 128, 131, 128, 130, 130, 130, 122, 127, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182423864Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182445665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182459355Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182472875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182485955Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182499126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"":"downstairs","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182512726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182524956Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"0","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182538257Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"1","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182551497Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"client":"2","":"downstairs","session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182564847Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182577077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"35f06a25-ba31-4774-8160-540ba4fce409 is now active with session: 076870a5-f51e-4d96-9bbb-b0ced4a91763","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182589958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182601988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710,"session_id":"076870a5-f51e-4d96-9bbb-b0ced4a91763"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:30.182616649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 16 len 8192 data: 11 10 03/30 Flush 04/30 Write block 309 len 32768 data: 8 10 9 8 10 11 12 13 05/30 Read block 10 len 20480 06/30 Read block 102 len 32768 07/30 Write block 22 len 20480 data: 13 12 13 16 16 08/30 Write block 96 len 28672 data: 11 9 12 14 16 15 12 09/30 Write block 400 len 32768 data: 9 11 15 17 14 11 10 13 10/30 Read block 326 len 20480 11/30 Read block 43 len 12288 12/30 Read block 221 len 32768 13/30 Flush 14/30 Write block 389 len 36864 data: 10 11 9 9 9 10 10 11 12 15/30 Write block 533 len 16384 data: 10 9 11 12 16/30 Write block 381 len 36864 data: 10 11 9 10 8 9 11 10 11 17/30 Read block 583 len 20480 18/30 Read block 134 len 4096 19/30 Read block 580 len 24576 20/30 Write block 575 len 4096 data: 14 21/30 Read block 304 len 40960 22/30 Read block 327 len 20480 23/30 Read block 292 len 24576 24/30 Read block 103 len 16384 25/30 Write block 43 len 16384 data: 13 12 13 15 26/30 Read block 382 len 40960 27/30 Read block 192 len 36864 28/30 Write block 27 len 32768 data: 12 11 9 12 13 11 11 12 29/30 Write block 338 len 20480 data: 11 10 11 9 8 30/30 Read block 434 len 24576 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 17 16 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:30.202443432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 225 225 225 F F F 1 020-039 132 132 132 226 226 226 T T T 2 040-059 128 128 128 218 218 218 T T T 3 060-079 128 128 128 218 218 218 F F F 4 080-099 132 132 132 226 226 226 F F F 5 100-119 132 132 132 226 226 226 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 127 127 127 217 217 217 F F F 11 220-239 130 130 130 221 221 221 F F F 12 240-259 131 131 131 223 223 223 F F F 13 260-279 130 130 130 220 220 220 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 132 132 132 226 226 226 F F F 16 320-339 130 130 130 220 220 220 T T T 17 340-359 129 129 129 219 219 219 T T T 18 360-379 131 131 131 223 223 223 F F F 19 380-399 126 126 126 215 215 215 T T T 20 400-419 132 132 132 226 226 226 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 130 130 130 221 221 221 F F F 26 520-539 129 129 129 219 219 219 T T T 27 540-559 129 129 129 219 219 219 F F F 28 560-579 126 126 126 216 216 216 T T T 29 580-599 128 128 128 218 218 218 F F F Max gen: 132, Max flush: 226 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 225 225 225 F F F 1 020-039 133 133 133 227 227 227 F F F 2 040-059 133 133 133 227 227 227 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 132 132 132 226 226 226 F F F 5 100-119 132 132 132 226 226 226 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 127 127 127 217 217 217 F F F 11 220-239 130 130 130 221 221 221 F F F 12 240-259 131 131 131 223 223 223 F F F 13 260-279 130 130 130 220 220 220 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 132 132 132 226 226 226 F F F 16 320-339 133 133 133 227 227 227 F F F 17 340-359 133 133 133 227 227 227 F F F 18 360-379 131 131 131 223 223 223 F F F 19 380-399 133 133 133 227 227 227 F F F 20 400-419 132 132 132 226 226 226 F F F 21 420-439 126 126 126 216 216 216 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 130 130 130 221 221 221 F F F 26 520-539 133 133 133 227 227 227 F F F 27 540-559 129 129 129 219 219 219 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 128 128 128 218 218 218 F F F Max gen: 133, Max flush: 227 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.476568751Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:34.517706684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.518070163Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.548409009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.548950391Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.548983542Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.549001243Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.549024703Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718} {"msg":"Crucible 41ed9d70-88f4-4247-a262-13f53f84ae1b has session id: c889193f-b4eb-4f9d-9187-ac6dd7fdcc45","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.549415482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Upstairs opts: Upstairs UUID: 41ed9d70-88f4-4247-a262-13f53f84ae1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.549454463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Crucible stats registered with UUID: 41ed9d70-88f4-4247-a262-13f53f84ae1b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.549470784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579054471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579092912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718} {"msg":"41ed9d70-88f4-4247-a262-13f53f84ae1b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579179474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579197805Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579221245Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579236025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579250026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579263916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579285276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579312007Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579339288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579368189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_connection connected from Ok(127.0.0.1:37883)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579395769Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_connection connected from Ok(127.0.0.1:37560)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.57942567Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_connection connected from Ok(127.0.0.1:54576)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.579454Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.580104846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.580135377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.58026804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.580311201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.580345832Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.580379793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.580792252Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.580827364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.58110549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581382987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581408117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581422558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581493959Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.58151307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.58152687Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581564091Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581580811Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[0]R flush_numbers[0..12]: [225, 227, 227, 218, 226, 226, 221, 221, 221, 209, 217, 221]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581601292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[0]R generation[0..12]: [132, 133, 133, 128, 132, 132, 130, 130, 130, 122, 127, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581615552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581629763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[1]R flush_numbers[0..12]: [225, 227, 227, 218, 226, 226, 221, 221, 221, 209, 217, 221]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581643823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[1]R generation[0..12]: [132, 133, 133, 128, 132, 132, 130, 130, 130, 122, 127, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581657673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581674834Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[2]R flush_numbers[0..12]: [225, 227, 227, 218, 226, 226, 221, 221, 221, 209, 217, 221]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581689054Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[2]R generation[0..12]: [132, 133, 133, 128, 132, 132, 130, 130, 130, 122, 127, 130]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581702894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581716855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581729895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581743145Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581756226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:12:34.581769276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"":"downstairs","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581786856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581798957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"0","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581812447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"1","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581825607Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"client":"2","":"downstairs","session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581839678Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581852328Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"41ed9d70-88f4-4247-a262-13f53f84ae1b is now active with session: c889193f-b4eb-4f9d-9187-ac6dd7fdcc45","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581865128Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581879298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718,"session_id":"c889193f-b4eb-4f9d-9187-ac6dd7fdcc45"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:34.581892029Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 203 len 16384 data: 10 6 8 8 02/30 Write block 220 len 32768 data: 12 12 13 13 13 13 10 10 03/30 Read block 402 len 40960 04/30 Write block 415 len 28672 data: 11 10 11 9 9 9 10 05/30 Read block 97 len 16384 06/30 Read block 25 len 8192 07/30 Flush 08/30 Write block 202 len 40960 data: 10 11 7 9 9 9 10 9 7 8 09/30 Write block 265 len 24576 data: 7 9 13 13 17 16 10/30 Read block 406 len 12288 11/30 Read block 2 len 24576 12/30 Write block 400 len 16384 data: 10 12 16 18 13/30 Flush 14/30 Flush 15/30 Write block 38 len 32768 data: 13 11 10 12 13 14 13 14 16/30 Write block 173 len 16384 data: 11 11 10 11 17/30 Flush 18/30 Write block 257 len 40960 data: 9 10 9 11 9 9 8 7 8 10 19/30 Read block 51 len 24576 20/30 Read block 378 len 16384 21/30 Read block 143 len 8192 22/30 Read block 216 len 36864 23/30 Read block 452 len 12288 24/30 Flush 25/30 Write block 539 len 28672 data: 13 16 13 13 14 14 12 26/30 Read block 495 len 20480 27/30 Write block 314 len 20480 data: 12 13 14 12 12 28/30 Write block 99 len 28672 data: 15 17 16 13 11 12 11 29/30 Read block 27 len 40960 30/30 Write block 36 len 36864 data: 13 13 14 12 11 13 14 15 14 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:34.602660085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 225 225 225 F F F 1 020-039 134 134 134 231 231 231 T T T 2 040-059 134 134 134 231 231 231 T T T 3 060-079 128 128 128 218 218 218 F F F 4 080-099 132 132 132 226 226 226 T T T 5 100-119 132 132 132 226 226 226 T T T 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 134 134 134 231 231 231 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 134 134 134 228 228 228 F F F 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 132 132 132 226 226 226 T T T 16 320-339 133 133 133 227 227 227 F F F 17 340-359 133 133 133 227 227 227 F F F 18 360-379 131 131 131 223 223 223 F F F 19 380-399 133 133 133 227 227 227 F F F 20 400-419 134 134 134 229 229 229 F F F 21 420-439 134 134 134 228 228 228 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 130 130 130 221 221 221 F F F 26 520-539 133 133 133 227 227 227 T T T 27 540-559 129 129 129 219 219 219 T T T 28 560-579 133 133 133 227 227 227 F F F 29 580-599 128 128 128 218 218 218 F F F Max gen: 134, Max flush: 232 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 225 225 225 F F F 1 020-039 135 135 135 233 233 233 F F F 2 040-059 135 135 135 233 233 233 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 135 135 135 233 233 233 F F F 5 100-119 135 135 135 233 233 233 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 134 134 134 231 231 231 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 134 134 134 228 228 228 F F F 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 135 135 135 233 233 233 F F F 16 320-339 133 133 133 227 227 227 F F F 17 340-359 133 133 133 227 227 227 F F F 18 360-379 131 131 131 223 223 223 F F F 19 380-399 133 133 133 227 227 227 F F F 20 400-419 134 134 134 229 229 229 F F F 21 420-439 134 134 134 228 228 228 F F F 22 440-459 126 126 126 213 213 213 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 122 122 122 208 208 208 F F F 25 500-519 130 130 130 221 221 221 F F F 26 520-539 135 135 135 233 233 233 F F F 27 540-559 135 135 135 233 233 233 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 128 128 128 218 218 218 F F F Max gen: 135, Max flush: 233 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.901975636Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.94440565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.944980954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.945017615Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.945032475Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.945051126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726} {"msg":"Crucible 9f92f826-f514-44ee-9536-1c21ebe8f044 has session id: b0175137-0502-402f-b03a-7fce6b399ec3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.945472816Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Upstairs opts: Upstairs UUID: 9f92f826-f514-44ee-9536-1c21ebe8f044, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.945521887Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Crucible stats registered with UUID: 9f92f826-f514-44ee-9536-1c21ebe8f044","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.945546537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.97495874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.974991861Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726} {"msg":"9f92f826-f514-44ee-9536-1c21ebe8f044 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975063753Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975086673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975117144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975135414Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975152765Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975183926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975217647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975249187Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975287108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975329439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35221)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.97536379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37094)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975430272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45643)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.975465562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976106708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976136649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976152839Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976167179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976181419Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.97620335Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976780694Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976812435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.976829745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977358728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977387959Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977406869Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.97747231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977492941Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977506981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977519431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977533132Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[0]R flush_numbers[0..12]: [225, 233, 233, 218, 233, 233, 221, 221, 231, 209, 229, 228]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977547192Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[0]R generation[0..12]: [132, 135, 135, 128, 135, 135, 130, 130, 134, 122, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977561323Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977575623Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[1]R flush_numbers[0..12]: [225, 233, 233, 218, 233, 233, 221, 221, 231, 209, 229, 228]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977589643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[1]R generation[0..12]: [132, 135, 135, 128, 135, 135, 130, 130, 134, 122, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977603873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977617974Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[2]R flush_numbers[0..12]: [225, 233, 233, 218, 233, 233, 221, 221, 231, 209, 229, 228]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977631934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[2]R generation[0..12]: [132, 135, 135, 128, 135, 135, 130, 130, 134, 122, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977645774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977662635Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977676455Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977690016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Next flush: 234","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:12:38.977703416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977796308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"":"downstairs","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977818759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977831669Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"0","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.97785253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"1","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.97787856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"client":"2","":"downstairs","session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977906391Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977922401Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"9f92f826-f514-44ee-9536-1c21ebe8f044 is now active with session: b0175137-0502-402f-b03a-7fce6b399ec3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977936912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977959562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726,"session_id":"b0175137-0502-402f-b03a-7fce6b399ec3"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:38.977984083Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 28672 data: 10 10 10 9 8 11 11 02/30 Flush 03/30 Flush 04/30 Write block 154 len 28672 data: 17 20 20 18 20 22 18 05/30 Flush 06/30 Write block 421 len 16384 data: 11 13 12 11 07/30 Read block 359 len 28672 08/30 Read block 389 len 4096 09/30 Read block 206 len 12288 10/30 Flush 11/30 Write block 7 len 12288 data: 9 8 11 12/30 Read block 339 len 20480 13/30 Read block 271 len 24576 14/30 Write block 51 len 20480 data: 13 14 13 13 16 15/30 Write block 92 len 12288 data: 13 15 11 16/30 Write block 111 len 28672 data: 5 5 6 6 7 10 11 17/30 Write block 456 len 16384 data: 11 9 10 11 18/30 Write block 24 len 32768 data: 14 17 17 13 12 10 13 14 19/30 Read block 262 len 24576 20/30 Read block 325 len 32768 21/30 Write block 588 len 40960 data: 8 10 9 9 9 8 8 7 8 7 22/30 Read block 16 len 8192 23/30 Read block 186 len 24576 24/30 Read block 17 len 16384 25/30 Read block 67 len 4096 26/30 Write block 49 len 4096 data: 14 27/30 Write block 469 len 32768 data: 8 9 9 7 8 4 4 5 28/30 Write block 513 len 28672 data: 9 8 12 11 11 9 7 29/30 Write block 221 len 40960 data: 13 14 14 14 14 11 11 8 9 8 30/30 Read block 369 len 36864 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 3 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 18 20 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:38.997198022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 225 225 225 T T T 1 020-039 135 135 135 233 233 233 T T T 2 040-059 135 135 135 233 233 233 T T T 3 060-079 128 128 128 218 218 218 F F F 4 080-099 135 135 135 233 233 233 T T T 5 100-119 135 135 135 233 233 233 T T T 6 120-139 130 130 130 221 221 221 F F F 7 140-159 136 136 136 236 236 236 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 134 134 134 228 228 228 T T T 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 135 135 135 233 233 233 F F F 16 320-339 133 133 133 227 227 227 F F F 17 340-359 133 133 133 227 227 227 F F F 18 360-379 131 131 131 223 223 223 F F F 19 380-399 133 133 133 227 227 227 F F F 20 400-419 134 134 134 229 229 229 F F F 21 420-439 136 136 136 237 237 237 F F F 22 440-459 126 126 126 213 213 213 T T T 23 460-479 126 126 126 213 213 213 T T T 24 480-499 136 136 136 234 234 234 F F F 25 500-519 130 130 130 221 221 221 T T T 26 520-539 135 135 135 233 233 233 F F F 27 540-559 135 135 135 233 233 233 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 128 128 128 218 218 218 T T T Max gen: 136, Max flush: 237 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 137 137 137 238 238 238 F F F 2 040-059 137 137 137 238 238 238 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 137 137 137 238 238 238 F F F 5 100-119 137 137 137 238 238 238 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 136 136 136 236 236 236 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 137 137 137 238 238 238 F F F 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 135 135 135 233 233 233 F F F 16 320-339 133 133 133 227 227 227 F F F 17 340-359 133 133 133 227 227 227 F F F 18 360-379 131 131 131 223 223 223 F F F 19 380-399 133 133 133 227 227 227 F F F 20 400-419 134 134 134 229 229 229 F F F 21 420-439 136 136 136 237 237 237 F F F 22 440-459 137 137 137 238 238 238 F F F 23 460-479 137 137 137 238 238 238 F F F 24 480-499 136 136 136 234 234 234 F F F 25 500-519 137 137 137 238 238 238 F F F 26 520-539 135 135 135 233 233 233 F F F 27 540-559 135 135 135 233 233 233 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 137, Max flush: 238 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.285263844Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:43.327490004Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.327853492Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.358873444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.359420577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.359455548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.359473038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.359488089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734} {"msg":"Crucible e2a45127-5ef2-4fb6-b589-00ba29c0ee00 has session id: b62a69c7-d7df-4d4d-afe8-1231ed3a1075","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.359881988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Upstairs opts: Upstairs UUID: e2a45127-5ef2-4fb6-b589-00ba29c0ee00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.359924889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Crucible stats registered with UUID: e2a45127-5ef2-4fb6-b589-00ba29c0ee00","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.359949929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389597808Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389637289Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734} {"msg":"e2a45127-5ef2-4fb6-b589-00ba29c0ee00 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.38966733Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389689691Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389735301Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389774413Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389813703Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389851724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389887105Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.389926656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390165232Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390207193Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_connection connected from Ok(127.0.0.1:43672)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390234584Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_connection connected from Ok(127.0.0.1:48748)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390258474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_connection connected from Ok(127.0.0.1:64149)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390283465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390659534Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390690055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390706225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390736905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.390890169Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.39093843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.391455283Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.391486094Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.391507614Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392050537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392083368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392098968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392111558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392125109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392139199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392151609Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.39216524Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[0]R flush_numbers[0..12]: [238, 238, 238, 218, 238, 238, 221, 236, 236, 209, 229, 238]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.39217945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[0]R generation[0..12]: [137, 137, 137, 128, 137, 137, 130, 136, 136, 122, 134, 137]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.39219361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392207981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[1]R flush_numbers[0..12]: [238, 238, 238, 218, 238, 238, 221, 236, 236, 209, 229, 238]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392222101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[1]R generation[0..12]: [137, 137, 137, 128, 137, 137, 130, 136, 136, 122, 134, 137]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392236201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392250452Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[2]R flush_numbers[0..12]: [238, 238, 238, 218, 238, 238, 221, 236, 236, 209, 229, 238]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392264322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[2]R generation[0..12]: [137, 137, 137, 128, 137, 137, 130, 136, 136, 122, 134, 137]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392280213Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392294353Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:12:43.392307583Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392325063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392338194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392351404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"":"downstairs","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392365144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392377225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"0","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392390815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"1","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392404295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"client":"2","":"downstairs","session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392417666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392430056Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"e2a45127-5ef2-4fb6-b589-00ba29c0ee00 is now active with session: b62a69c7-d7df-4d4d-afe8-1231ed3a1075","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392443006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392455177Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734,"session_id":"b62a69c7-d7df-4d4d-afe8-1231ed3a1075"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:43.392467527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 548 len 28672 data: 10 7 10 10 14 11 13 02/30 Flush 03/30 Write block 287 len 8192 data: 15 13 04/30 Read block 141 len 8192 05/30 Write block 38 len 12288 data: 15 13 12 06/30 Write block 129 len 16384 data: 8 8 6 6 07/30 Write block 342 len 40960 data: 9 8 11 12 16 17 15 15 13 12 08/30 Read block 64 len 36864 09/30 Read block 584 len 8192 10/30 Read block 438 len 24576 11/30 Flush 12/30 Write block 280 len 12288 data: 6 5 5 13/30 Read block 288 len 8192 14/30 Read block 388 len 20480 15/30 Read block 112 len 12288 16/30 Read block 581 len 8192 17/30 Write block 139 len 16384 data: 9 13 16 14 18/30 Write block 110 len 16384 data: 7 6 6 7 19/30 Read block 21 len 12288 20/30 Read block 232 len 40960 21/30 Write block 234 len 12288 data: 10 10 11 22/30 Read block 587 len 36864 23/30 Write block 440 len 12288 data: 7 7 7 24/30 Flush 25/30 Write block 367 len 24576 data: 10 12 13 9 8 8 26/30 Read block 419 len 24576 27/30 Read block 402 len 24576 28/30 Write block 237 len 36864 data: 11 13 11 9 11 11 9 8 8 29/30 Write block 147 len 24576 data: 9 7 7 7 7 11 30/30 Read block 6 len 32768 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 6 6 4 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:43.411315377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 137 137 137 238 238 238 F F F 5 100-119 138 138 138 241 241 241 F F F 6 120-139 138 138 138 241 241 241 F F F 7 140-159 138 138 138 241 241 241 T T T 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 138 138 138 241 241 241 T T T 12 240-259 134 134 134 232 232 232 T T T 13 260-279 134 134 134 232 232 232 F F F 14 280-299 138 138 138 241 241 241 F F F 15 300-319 135 135 135 233 233 233 F F F 16 320-339 133 133 133 227 227 227 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 131 131 131 223 223 223 T T T 19 380-399 133 133 133 227 227 227 F F F 20 400-419 134 134 134 229 229 229 F F F 21 420-439 136 136 136 237 237 237 F F F 22 440-459 138 138 138 241 241 241 F F F 23 460-479 137 137 137 238 238 238 F F F 24 480-499 136 136 136 234 234 234 F F F 25 500-519 137 137 137 238 238 238 F F F 26 520-539 135 135 135 233 233 233 F F F 27 540-559 138 138 138 239 239 239 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 138, Max flush: 241 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 137 137 137 238 238 238 F F F 5 100-119 138 138 138 241 241 241 F F F 6 120-139 138 138 138 241 241 241 F F F 7 140-159 139 139 139 242 242 242 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 139 139 139 242 242 242 F F F 12 240-259 139 139 139 242 242 242 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 138 138 138 241 241 241 F F F 15 300-319 135 135 135 233 233 233 F F F 16 320-339 133 133 133 227 227 227 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 139 139 139 242 242 242 F F F 19 380-399 133 133 133 227 227 227 F F F 20 400-419 134 134 134 229 229 229 F F F 21 420-439 136 136 136 237 237 237 F F F 22 440-459 138 138 138 241 241 241 F F F 23 460-479 137 137 137 238 238 238 F F F 24 480-499 136 136 136 234 234 234 F F F 25 500-519 137 137 137 238 238 238 F F F 26 520-539 135 135 135 233 233 233 F F F 27 540-559 138 138 138 239 239 239 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 139, Max flush: 242 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.692215998Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.735763579Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.736424155Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.736461006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.736476176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.736490646Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742} {"msg":"Crucible 0060dd71-d40c-40d8-ae57-5ac901a0bb81 has session id: e46ac5f1-1fbe-4951-b665-cb7641a5b43f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.736893186Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Upstairs opts: Upstairs UUID: 0060dd71-d40c-40d8-ae57-5ac901a0bb81, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.736931897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Crucible stats registered with UUID: 0060dd71-d40c-40d8-ae57-5ac901a0bb81","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.736948828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767333404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767377865Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742} {"msg":"0060dd71-d40c-40d8-ae57-5ac901a0bb81 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767441576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767468627Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767502478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767531668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767560989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.76758782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.76761741Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767647871Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767829556Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767868356Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37143)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767900007Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51174)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767932088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35907)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.767965049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.768515242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.768552033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.768583424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.768613684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.768640015Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.768655065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769098386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769142777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.76925056Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769582157Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769612748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769638669Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769718851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769739161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769756642Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769781712Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769796013Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[0]R flush_numbers[0..12]: [238, 240, 240, 218, 238, 241, 241, 242, 236, 209, 229, 242]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769834314Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[0]R generation[0..12]: [137, 138, 138, 128, 137, 138, 138, 139, 136, 122, 134, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769852364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769866864Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[1]R flush_numbers[0..12]: [238, 240, 240, 218, 238, 241, 241, 242, 236, 209, 229, 242]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769880905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[1]R generation[0..12]: [137, 138, 138, 128, 137, 138, 138, 139, 136, 122, 134, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769894695Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769908865Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[2]R flush_numbers[0..12]: [238, 240, 240, 218, 238, 241, 241, 242, 236, 209, 229, 242]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769922806Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[2]R generation[0..12]: [137, 138, 138, 128, 137, 138, 138, 139, 136, 122, 134, 139]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769936616Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769950536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769963567Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.769977007Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:12:47.769993047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.770034198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"":"downstairs","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.770060999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.77008458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"0","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.77010887Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"1","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.770130881Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"client":"2","":"downstairs","session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:12:47.770155331Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.770176702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"0060dd71-d40c-40d8-ae57-5ac901a0bb81 is now active with session: e46ac5f1-1fbe-4951-b665-cb7641a5b43f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.770200562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.770214893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742,"session_id":"e46ac5f1-1fbe-4951-b665-cb7641a5b43f"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:47.770227423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 241 len 28672 data: 12 12 10 9 9 7 6 02/30 Write block 387 len 4096 data: 12 03/30 Read block 165 len 20480 04/30 Read block 140 len 16384 05/30 Write block 263 len 8192 data: 9 8 06/30 Write block 113 len 8192 data: 8 7 07/30 Write block 222 len 20480 data: 15 15 15 15 12 08/30 Read block 287 len 36864 09/30 Write block 531 len 8192 data: 9 11 10/30 Read block 418 len 36864 11/30 Read block 20 len 20480 12/30 Write block 127 len 8192 data: 14 9 13/30 Read block 228 len 24576 14/30 Read block 465 len 32768 15/30 Write block 277 len 36864 data: 5 5 6 7 6 6 8 10 11 16/30 Read block 238 len 20480 17/30 Read block 16 len 8192 18/30 Write block 289 len 32768 data: 13 14 16 19 14 14 15 15 19/30 Write block 512 len 20480 data: 9 10 9 13 12 20/30 Read block 168 len 20480 21/30 Write block 557 len 4096 data: 14 22/30 Write block 328 len 24576 data: 11 10 9 8 11 12 23/30 Read block 499 len 8192 24/30 Read block 101 len 24576 25/30 Read block 415 len 16384 26/30 Read block 559 len 32768 27/30 Read block 251 len 20480 28/30 Read block 228 len 4096 29/30 Write block 495 len 32768 data: 12 10 9 8 9 10 13 13 30/30 Write block 391 len 24576 data: 10 10 10 11 11 12 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:47.788298444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 137 137 137 238 238 238 F F F 5 100-119 138 138 138 241 241 241 T T T 6 120-139 138 138 138 241 241 241 T T T 7 140-159 139 139 139 242 242 242 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 139 139 139 242 242 242 T T T 12 240-259 139 139 139 242 242 242 T T T 13 260-279 134 134 134 232 232 232 T T T 14 280-299 138 138 138 241 241 241 T T T 15 300-319 135 135 135 233 233 233 F F F 16 320-339 133 133 133 227 227 227 T T T 17 340-359 138 138 138 240 240 240 F F F 18 360-379 139 139 139 242 242 242 F F F 19 380-399 133 133 133 227 227 227 T T T 20 400-419 134 134 134 229 229 229 F F F 21 420-439 136 136 136 237 237 237 F F F 22 440-459 138 138 138 241 241 241 F F F 23 460-479 137 137 137 238 238 238 F F F 24 480-499 136 136 136 234 234 234 T T T 25 500-519 137 137 137 238 238 238 T T T 26 520-539 135 135 135 233 233 233 T T T 27 540-559 138 138 138 239 239 239 T T T 28 560-579 133 133 133 227 227 227 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 139, Max flush: 242 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 137 137 137 238 238 238 F F F 5 100-119 140 140 140 243 243 243 F F F 6 120-139 140 140 140 243 243 243 F F F 7 140-159 139 139 139 242 242 242 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 140 140 140 243 243 243 F F F 15 300-319 135 135 135 233 233 233 F F F 16 320-339 140 140 140 243 243 243 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 139 139 139 242 242 242 F F F 19 380-399 140 140 140 243 243 243 F F F 20 400-419 134 134 134 229 229 229 F F F 21 420-439 136 136 136 237 237 237 F F F 22 440-459 138 138 138 241 241 241 F F F 23 460-479 137 137 137 238 238 238 F F F 24 480-499 140 140 140 243 243 243 F F F 25 500-519 140 140 140 243 243 243 F F F 26 520-539 140 140 140 243 243 243 F F F 27 540-559 140 140 140 243 243 243 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 140, Max flush: 243 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.091257242Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.134317172Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.134949827Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.134985358Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.135000038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.135015048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750} {"msg":"Crucible dcbfcbce-741e-4f6d-883d-137d980236c9 has session id: 20b6f468-f95a-4875-931a-38fdc68bcd3e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.135449279Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Upstairs opts: Upstairs UUID: dcbfcbce-741e-4f6d-883d-137d980236c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.13549655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Crucible stats registered with UUID: dcbfcbce-741e-4f6d-883d-137d980236c9","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.135519291Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165242101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165279142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750} {"msg":"dcbfcbce-741e-4f6d-883d-137d980236c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165339253Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165367254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165393845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165410375Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165436936Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165464246Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165491897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165519498Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165545668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165572199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60544)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.16560588Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65460)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165790484Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37480)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.165821005Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.166300796Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.166326947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.166512941Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.166545612Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.166563443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.166588503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167058574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167090425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167222308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167644048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167672049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167688009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.16770417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.16771843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.16773213Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167745411Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167759051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[0]R flush_numbers[0..12]: [238, 240, 240, 218, 238, 243, 243, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167773122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[0]R generation[0..12]: [137, 138, 138, 128, 137, 140, 140, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167787272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167801872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[1]R flush_numbers[0..12]: [238, 240, 240, 218, 238, 243, 243, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167816023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[1]R generation[0..12]: [137, 138, 138, 128, 137, 140, 140, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167829773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167843843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[2]R flush_numbers[0..12]: [238, 240, 240, 218, 238, 243, 243, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167857773Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[2]R generation[0..12]: [137, 138, 138, 128, 137, 140, 140, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167871634Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167885684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167900825Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167914285Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167927475Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:12:52.167940675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"":"downstairs","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167961936Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167974006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"0","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.167987127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"1","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.168000307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"client":"2","":"downstairs","session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.168013617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.168026077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"dcbfcbce-741e-4f6d-883d-137d980236c9 is now active with session: 20b6f468-f95a-4875-931a-38fdc68bcd3e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.168038938Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.168050948Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750,"session_id":"20b6f468-f95a-4875-931a-38fdc68bcd3e"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:52.168063209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 106 len 8192 data: 9 8 02/30 Read block 578 len 8192 03/30 Write block 503 len 28672 data: 15 13 12 14 15 11 10 04/30 Read block 21 len 20480 05/30 Flush 06/30 Flush 07/30 Write block 313 len 32768 data: 11 13 14 15 13 13 13 12 08/30 Write block 95 len 12288 data: 12 12 10 09/30 Write block 456 len 24576 data: 12 10 11 12 10 10 10/30 Flush 11/30 Write block 128 len 20480 data: 10 9 9 7 7 12/30 Write block 590 len 36864 data: 10 10 10 9 9 8 9 8 6 13/30 Flush 14/30 Write block 548 len 40960 data: 11 8 11 11 15 12 14 11 11 15 15/30 Write block 385 len 36864 data: 9 10 13 11 12 12 11 11 11 16/30 Read block 238 len 8192 17/30 Write block 408 len 16384 data: 15 13 11 11 18/30 Write block 282 len 20480 data: 7 9 11 12 14 19/30 Read block 503 len 4096 20/30 Write block 418 len 16384 data: 10 10 10 12 21/30 Read block 323 len 24576 22/30 Read block 375 len 20480 23/30 Read block 212 len 40960 24/30 Write block 446 len 24576 data: 7 6 6 7 8 8 25/30 Write block 542 len 4096 data: 14 26/30 Read block 59 len 4096 27/30 Write block 409 len 40960 data: 14 12 12 13 13 12 12 11 12 11 28/30 Read block 443 len 20480 29/30 Write block 68 len 28672 data: 13 12 11 8 7 8 7 30/30 Read block 231 len 16384 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 16 16 17 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:52.188072027Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 128 128 128 218 218 218 T T T 4 080-099 142 142 142 246 246 246 F F F 5 100-119 142 142 142 244 244 244 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 139 139 139 242 242 242 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 140 140 140 243 243 243 T T T 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 246 246 246 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 139 139 139 242 242 242 F F F 19 380-399 140 140 140 243 243 243 T T T 20 400-419 134 134 134 229 229 229 T T T 21 420-439 136 136 136 237 237 237 T T T 22 440-459 142 142 142 246 246 246 T T T 23 460-479 142 142 142 246 246 246 F F F 24 480-499 140 140 140 243 243 243 F F F 25 500-519 142 142 142 244 244 244 F F F 26 520-539 140 140 140 243 243 243 F F F 27 540-559 140 140 140 243 243 243 T T T 28 560-579 133 133 133 227 227 227 F F F 29 580-599 142 142 142 247 247 247 F F F Max gen: 142, Max flush: 247 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 143 143 143 248 248 248 F F F 4 080-099 142 142 142 246 246 246 F F F 5 100-119 142 142 142 244 244 244 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 139 139 139 242 242 242 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 143 143 143 248 248 248 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 246 246 246 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 139 139 139 242 242 242 F F F 19 380-399 143 143 143 248 248 248 F F F 20 400-419 143 143 143 248 248 248 F F F 21 420-439 143 143 143 248 248 248 F F F 22 440-459 143 143 143 248 248 248 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 140 140 140 243 243 243 F F F 25 500-519 142 142 142 244 244 244 F F F 26 520-539 140 140 140 243 243 243 F F F 27 540-559 143 143 143 248 248 248 F F F 28 560-579 133 133 133 227 227 227 F F F 29 580-599 142 142 142 247 247 247 F F F Max gen: 143, Max flush: 248 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.462133134Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.506392822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.506971806Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.507008667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.507023127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.507038117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758} {"msg":"Crucible a359dad9-842d-4f23-b89a-dd8c14dcef30 has session id: db2229ac-15de-41a5-b9eb-4240430d3a25","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.507491508Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Upstairs opts: Upstairs UUID: a359dad9-842d-4f23-b89a-dd8c14dcef30, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.5075417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Crucible stats registered with UUID: a359dad9-842d-4f23-b89a-dd8c14dcef30","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.50756817Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.537837134Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.537865134Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758} {"msg":"a359dad9-842d-4f23-b89a-dd8c14dcef30 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.537944206Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.537972667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538014738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538032748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538047149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538061009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538074749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.53809645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.5381222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538149431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_connection connected from Ok(127.0.0.1:61487)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538204982Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_connection connected from Ok(127.0.0.1:36053)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538242363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_connection connected from Ok(127.0.0.1:35956)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.538273794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539107784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539149985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539182366Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539215307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539245927Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539275388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539669377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539813021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.539934894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540233981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540262992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540279032Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540304193Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540322203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540336373Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540468327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540487337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[0]R flush_numbers[0..12]: [238, 240, 240, 248, 246, 244, 247, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540501428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[0]R generation[0..12]: [137, 138, 138, 143, 142, 142, 142, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540515508Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540529758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[1]R flush_numbers[0..12]: [238, 240, 240, 248, 246, 244, 247, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540543818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[1]R generation[0..12]: [137, 138, 138, 143, 142, 142, 142, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540558539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540572769Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[2]R flush_numbers[0..12]: [238, 240, 240, 248, 246, 244, 247, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.54058673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[2]R generation[0..12]: [137, 138, 138, 143, 142, 142, 142, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.54060066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.54061454Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.5406277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540645191Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540660261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540683272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"":"downstairs","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540708432Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540732513Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"0","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540747613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"1","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540760794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"client":"2","":"downstairs","session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540774254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540786594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"a359dad9-842d-4f23-b89a-dd8c14dcef30 is now active with session: db2229ac-15de-41a5-b9eb-4240430d3a25","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540799544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:12:56.540811695Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758,"session_id":"db2229ac-15de-41a5-b9eb-4240430d3a25"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:12:56.540828525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 571 len 16384 data: 12 13 12 14 03/30 Read block 116 len 36864 04/30 Write block 303 len 4096 data: 7 05/30 Write block 33 len 28672 data: 12 13 11 14 14 16 14 06/30 Flush 07/30 Write block 311 len 32768 data: 10 9 12 14 15 16 14 14 08/30 Flush 09/30 Write block 361 len 36864 data: 11 11 12 10 10 11 11 13 14 10/30 Write block 575 len 40960 data: 15 13 12 13 12 12 11 9 10 9 11/30 Read block 130 len 8192 12/30 Read block 522 len 20480 13/30 Read block 103 len 20480 14/30 Read block 170 len 20480 15/30 Write block 449 len 24576 data: 8 9 9 9 10 11 16/30 Read block 80 len 24576 17/30 Write block 73 len 24576 data: 9 8 6 5 9 8 18/30 Read block 318 len 28672 19/30 Read block 74 len 20480 20/30 Write block 375 len 20480 data: 6 8 11 13 12 21/30 Write block 497 len 32768 data: 10 9 10 11 14 14 16 14 22/30 Read block 209 len 20480 23/30 Write block 409 len 12288 data: 15 13 13 24/30 Read block 269 len 16384 25/30 Write block 370 len 16384 data: 10 9 9 7 26/30 Write block 315 len 16384 data: 16 17 15 15 27/30 Write block 372 len 12288 data: 10 8 7 28/30 Write block 538 len 4096 data: 13 29/30 Read block 473 len 32768 30/30 Write block 24 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 9 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:12:56.559641075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 144 144 144 250 250 250 T T T 2 040-059 138 138 138 240 240 240 F F F 3 060-079 143 143 143 248 248 248 T T T 4 080-099 142 142 142 246 246 246 F F F 5 100-119 142 142 142 244 244 244 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 139 139 139 242 242 242 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 143 143 143 248 248 248 F F F 15 300-319 144 144 144 251 251 251 T T T 16 320-339 142 142 142 246 246 246 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 139 139 139 242 242 242 T T T 19 380-399 143 143 143 248 248 248 F F F 20 400-419 143 143 143 248 248 248 T T T 21 420-439 143 143 143 248 248 248 F F F 22 440-459 143 143 143 248 248 248 T T T 23 460-479 142 142 142 246 246 246 F F F 24 480-499 140 140 140 243 243 243 T T T 25 500-519 142 142 142 244 244 244 T T T 26 520-539 140 140 140 243 243 243 T T T 27 540-559 143 143 143 248 248 248 F F F 28 560-579 144 144 144 250 250 250 T T T 29 580-599 142 142 142 247 247 247 T T T Max gen: 144, Max flush: 251 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 145 145 145 252 252 252 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 145 145 145 252 252 252 F F F 4 080-099 142 142 142 246 246 246 F F F 5 100-119 142 142 142 244 244 244 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 139 139 139 242 242 242 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 143 143 143 248 248 248 F F F 15 300-319 145 145 145 252 252 252 F F F 16 320-339 142 142 142 246 246 246 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 145 145 145 252 252 252 F F F 19 380-399 143 143 143 248 248 248 F F F 20 400-419 145 145 145 252 252 252 F F F 21 420-439 143 143 143 248 248 248 F F F 22 440-459 145 145 145 252 252 252 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 145 145 145 252 252 252 F F F 25 500-519 145 145 145 252 252 252 F F F 26 520-539 145 145 145 252 252 252 F F F 27 540-559 143 143 143 248 248 248 F F F 28 560-579 145 145 145 252 252 252 F F F 29 580-599 145 145 145 252 252 252 F F F Max gen: 145, Max flush: 252 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.85623098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:00.898415629Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.898777668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.929934132Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.930528816Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.930563587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.930577408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.930592078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766} {"msg":"Crucible 3a02c097-d471-48c2-a44f-83ef050ccf94 has session id: 84f31773-8ead-4ccc-90c5-9fb144e87b5d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.930993527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Upstairs opts: Upstairs UUID: 3a02c097-d471-48c2-a44f-83ef050ccf94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.931030948Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Crucible stats registered with UUID: 3a02c097-d471-48c2-a44f-83ef050ccf94","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.931047959Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960493393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960524573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766} {"msg":"3a02c097-d471-48c2-a44f-83ef050ccf94 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960596985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960619856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960648996Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960666847Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960684117Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960700658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960717488Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960734889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960751189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960768949Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56644)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960838851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55653)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.960884102Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59171)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.961075857Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.961661441Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.961689831Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.961705822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.961719892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.961737993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.961791774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.962297496Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.962354447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.96244675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.962920501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.962947022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.962965102Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.962979392Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.962993643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963007103Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963019883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963033913Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[0]R flush_numbers[0..12]: [238, 252, 240, 252, 246, 244, 247, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963048094Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[0]R generation[0..12]: [137, 145, 138, 145, 142, 142, 142, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963062204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963076225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[1]R flush_numbers[0..12]: [238, 252, 240, 252, 246, 244, 247, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963090165Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[1]R generation[0..12]: [137, 145, 138, 145, 142, 142, 142, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963104035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963118206Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[2]R flush_numbers[0..12]: [238, 252, 240, 252, 246, 244, 247, 242, 236, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963132116Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[2]R generation[0..12]: [137, 145, 138, 145, 142, 142, 142, 139, 136, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963145946Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963160127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963175067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963188657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963201617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963222458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"":"downstairs","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963235838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963248039Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"0","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963261209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"1","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963274289Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"client":"2","":"downstairs","session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.96328749Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.96330002Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"3a02c097-d471-48c2-a44f-83ef050ccf94 is now active with session: 84f31773-8ead-4ccc-90c5-9fb144e87b5d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.96331296Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.9633252Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766,"session_id":"84f31773-8ead-4ccc-90c5-9fb144e87b5d"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:00.963337651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 425 len 36864 data: 13 13 13 14 12 13 10 9 8 02/30 Read block 223 len 16384 03/30 Write block 379 len 20480 data: 13 10 11 12 10 04/30 Write block 413 len 16384 data: 14 13 13 12 05/30 Write block 508 len 32768 data: 12 11 12 11 10 11 10 14 06/30 Read block 335 len 8192 07/30 Write block 147 len 40960 data: 10 8 8 8 8 12 15 18 21 21 08/30 Flush 09/30 Write block 434 len 36864 data: 10 8 7 5 5 8 8 8 8 10/30 Flush 11/30 Write block 156 len 40960 data: 22 19 21 23 19 19 21 16 16 15 12/30 Read block 152 len 32768 13/30 Write block 272 len 40960 data: 14 14 11 11 7 6 6 7 8 7 14/30 Write block 240 len 24576 data: 10 13 13 11 10 10 15/30 Read block 217 len 28672 16/30 Write block 437 len 28672 data: 6 6 9 9 9 9 7 17/30 Write block 36 len 8192 data: 15 15 18/30 Read block 392 len 4096 19/30 Read block 182 len 16384 20/30 Write block 435 len 40960 data: 9 8 7 7 10 10 10 10 8 7 21/30 Read block 277 len 28672 22/30 Read block 341 len 12288 23/30 Write block 586 len 12288 data: 9 10 9 24/30 Write block 88 len 16384 data: 8 11 12 12 25/30 Read block 336 len 4096 26/30 Flush 27/30 Write block 445 len 28672 data: 7 8 7 7 9 10 10 28/30 Write block 81 len 4096 data: 9 29/30 Write block 422 len 24576 data: 14 13 12 14 14 14 30/30 Read block 342 len 16384 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:00.98378781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 145 145 145 252 252 252 F F F 4 080-099 146 146 146 255 255 255 T T T 5 100-119 142 142 142 244 244 244 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 146 146 146 255 255 255 F F F 8 160-179 146 146 146 255 255 255 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 146 146 146 255 255 255 F F F 13 260-279 146 146 146 255 255 255 F F F 14 280-299 146 146 146 255 255 255 F F F 15 300-319 145 145 145 252 252 252 F F F 16 320-339 142 142 142 246 246 246 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 146 146 146 253 253 253 F F F 19 380-399 146 146 146 253 253 253 F F F 20 400-419 146 146 146 253 253 253 F F F 21 420-439 146 146 146 255 255 255 T T T 22 440-459 146 146 146 255 255 255 T T T 23 460-479 142 142 142 246 246 246 F F F 24 480-499 145 145 145 252 252 252 F F F 25 500-519 146 146 146 253 253 253 F F F 26 520-539 145 145 145 252 252 252 F F F 27 540-559 143 143 143 248 248 248 F F F 28 560-579 145 145 145 252 252 252 F F F 29 580-599 146 146 146 255 255 255 F F F Max gen: 146, Max flush: 255 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 F F F 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 145 145 145 252 252 252 F F F 4 080-099 147 147 147 256 256 256 F F F 5 100-119 142 142 142 244 244 244 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 146 146 146 255 255 255 F F F 8 160-179 146 146 146 255 255 255 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 146 146 146 255 255 255 F F F 13 260-279 146 146 146 255 255 255 F F F 14 280-299 146 146 146 255 255 255 F F F 15 300-319 145 145 145 252 252 252 F F F 16 320-339 142 142 142 246 246 246 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 146 146 146 253 253 253 F F F 19 380-399 146 146 146 253 253 253 F F F 20 400-419 146 146 146 253 253 253 F F F 21 420-439 147 147 147 256 256 256 F F F 22 440-459 147 147 147 256 256 256 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 145 145 145 252 252 252 F F F 25 500-519 146 146 146 253 253 253 F F F 26 520-539 145 145 145 252 252 252 F F F 27 540-559 143 143 143 248 248 248 F F F 28 560-579 145 145 145 252 252 252 F F F 29 580-599 146 146 146 255 255 255 F F F Max gen: 147, Max flush: 256 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.259879176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:05.300866055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.301219564Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.331458576Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.332056571Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.332090442Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.332104402Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.332118952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774} {"msg":"Crucible f0e98367-be37-4ec8-b7de-a36d7ca66634 has session id: dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.332604444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Upstairs opts: Upstairs UUID: f0e98367-be37-4ec8-b7de-a36d7ca66634, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.332656715Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Crucible stats registered with UUID: f0e98367-be37-4ec8-b7de-a36d7ca66634","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.332679926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362192201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362219412Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774} {"msg":"f0e98367-be37-4ec8-b7de-a36d7ca66634 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362301054Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362319684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362344055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362358775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362372755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362386326Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362399706Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362413187Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362427067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362517249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_connection connected from Ok(127.0.0.1:44887)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.36256303Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_connection connected from Ok(127.0.0.1:54656)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362609221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_connection connected from Ok(127.0.0.1:54545)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.362746304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.363280557Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.363309838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.363326378Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.36338873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.36341968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.363435221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.363886512Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364036665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364072886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364432375Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364469046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364485456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364502606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364520477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364534367Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364594249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364613789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[0]R flush_numbers[0..12]: [238, 255, 240, 252, 256, 244, 247, 255, 255, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.3646283Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[0]R generation[0..12]: [137, 146, 138, 145, 147, 142, 142, 146, 146, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.36464238Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.36465661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[1]R flush_numbers[0..12]: [238, 255, 240, 252, 256, 244, 247, 255, 255, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.36467063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[1]R generation[0..12]: [137, 146, 138, 145, 147, 142, 142, 146, 146, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364684581Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364698651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[2]R flush_numbers[0..12]: [238, 255, 240, 252, 256, 244, 247, 255, 255, 209, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364712351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[2]R generation[0..12]: [137, 146, 138, 145, 147, 142, 142, 146, 146, 122, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364726322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364740362Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364753503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364770023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:13:05.364783583Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364803374Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"":"downstairs","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364816484Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364828714Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"0","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364842114Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"1","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364855535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"client":"2","":"downstairs","session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364869275Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364882035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"f0e98367-be37-4ec8-b7de-a36d7ca66634 is now active with session: dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364895046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364911316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774,"session_id":"dac460a9-6fa8-4d6d-a52d-0f5a1c55ec95"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:05.364923727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 437 len 32768 data: 8 8 11 11 11 11 9 8 02/30 Read block 277 len 20480 03/30 Write block 480 len 16384 data: 13 17 15 13 04/30 Write block 260 len 40960 data: 12 10 10 10 9 9 11 14 14 18 05/30 Read block 12 len 16384 06/30 Read block 257 len 12288 07/30 Flush 08/30 Write block 547 len 20480 data: 10 12 9 12 12 09/30 Write block 280 len 36864 data: 9 8 8 10 12 13 15 16 14 10/30 Read block 259 len 20480 11/30 Read block 190 len 24576 12/30 Write block 107 len 20480 data: 9 7 8 8 7 13/30 Read block 439 len 36864 14/30 Write block 321 len 24576 data: 13 10 12 12 12 13 15/30 Write block 512 len 4096 data: 11 16/30 Read block 470 len 24576 17/30 Read block 466 len 40960 18/30 Read block 306 len 28672 19/30 Read block 543 len 4096 20/30 Write block 480 len 28672 data: 14 18 16 14 13 12 10 21/30 Flush 22/30 Write block 180 len 36864 data: 12 9 9 9 9 10 11 11 8 23/30 Read block 351 len 4096 24/30 Write block 95 len 24576 data: 13 13 11 13 16 18 25/30 Write block 153 len 40960 data: 16 19 22 23 20 22 24 20 20 22 26/30 Read block 348 len 8192 27/30 Read block 305 len 8192 28/30 Write block 17 len 12288 data: 11 9 9 29/30 Read block 424 len 16384 30/30 Read block 513 len 4096 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:05.385270023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 238 238 238 T T T 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 145 145 145 252 252 252 F F F 4 080-099 147 147 147 256 256 256 T T T 5 100-119 148 148 148 258 258 258 T T T 6 120-139 142 142 142 247 247 247 F F F 7 140-159 146 146 146 255 255 255 T T T 8 160-179 146 146 146 255 255 255 T T T 9 180-199 122 122 122 209 209 209 T T T 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 146 146 146 255 255 255 F F F 13 260-279 148 148 148 257 257 257 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 145 145 145 252 252 252 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 146 146 146 253 253 253 F F F 19 380-399 146 146 146 253 253 253 F F F 20 400-419 146 146 146 253 253 253 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 148 148 148 257 257 257 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 258 258 258 F F F 26 520-539 145 145 145 252 252 252 F F F 27 540-559 148 148 148 258 258 258 F F F 28 560-579 145 145 145 252 252 252 F F F 29 580-599 146 146 146 255 255 255 F F F Max gen: 148, Max flush: 258 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 259 259 259 F F F 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 145 145 145 252 252 252 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 149 149 149 259 259 259 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 149 149 149 259 259 259 F F F 8 160-179 149 149 149 259 259 259 F F F 9 180-199 149 149 149 259 259 259 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 146 146 146 255 255 255 F F F 13 260-279 148 148 148 257 257 257 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 145 145 145 252 252 252 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 146 146 146 253 253 253 F F F 19 380-399 146 146 146 253 253 253 F F F 20 400-419 146 146 146 253 253 253 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 148 148 148 257 257 257 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 258 258 258 F F F 26 520-539 145 145 145 252 252 252 F F F 27 540-559 148 148 148 258 258 258 F F F 28 560-579 145 145 145 252 252 252 F F F 29 580-599 146 146 146 255 255 255 F F F Max gen: 149, Max flush: 259 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.68651195Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.729505668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.730134153Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.730171514Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.730186274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.730201384Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782} {"msg":"Crucible 930d8079-24a3-4fc1-abac-6d6a8d34e4e4 has session id: b684578c-5f3b-4c48-9155-1b6813f0594e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.730637875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Upstairs opts: Upstairs UUID: 930d8079-24a3-4fc1-abac-6d6a8d34e4e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.730685426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Crucible stats registered with UUID: 930d8079-24a3-4fc1-abac-6d6a8d34e4e4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.730717147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761303728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761348359Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782} {"msg":"930d8079-24a3-4fc1-abac-6d6a8d34e4e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.76141357Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761445981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761489152Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761521423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761551354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761580174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761626966Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761657066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761688067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761716628Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52172)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761744038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60342)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761772739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49800)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.761887802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.762538327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.762576368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.762606169Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.76263502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.762698841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.762737042Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763156212Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763308126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763340396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763657964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763686055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763701715Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.76392189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763961991Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.763988702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764012452Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764043873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[0]R flush_numbers[0..12]: [259, 255, 240, 252, 259, 259, 247, 259, 259, 259, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764070874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[0]R generation[0..12]: [149, 146, 138, 145, 149, 149, 142, 149, 149, 149, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764097705Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764123225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[1]R flush_numbers[0..12]: [259, 255, 240, 252, 259, 259, 247, 259, 259, 259, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764148276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[1]R generation[0..12]: [149, 146, 138, 145, 149, 149, 142, 149, 149, 149, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764175147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764202807Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[2]R flush_numbers[0..12]: [259, 255, 240, 252, 259, 259, 247, 259, 259, 259, 229, 243]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764228528Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[2]R generation[0..12]: [149, 146, 138, 145, 149, 149, 142, 149, 149, 149, 134, 140]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764256318Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764295649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.76432665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764355051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764383772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764429762Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"":"downstairs","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764466334Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764498074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"0","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764519965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"1","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764534655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"client":"2","":"downstairs","session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764548465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764561426Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"930d8079-24a3-4fc1-abac-6d6a8d34e4e4 is now active with session: b684578c-5f3b-4c48-9155-1b6813f0594e","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764574636Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764587056Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782,"session_id":"b684578c-5f3b-4c48-9155-1b6813f0594e"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:09.764599657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 410 len 16384 data: 14 14 14 15 02/30 Read block 475 len 20480 03/30 Write block 61 len 24576 data: 14 13 12 14 17 19 04/30 Flush 05/30 Write block 70 len 8192 data: 12 9 06/30 Read block 542 len 40960 07/30 Read block 247 len 32768 08/30 Read block 10 len 28672 09/30 Flush 10/30 Write block 235 len 36864 data: 11 12 12 14 12 11 14 14 12 11/30 Write block 428 len 32768 data: 15 13 14 11 10 9 11 10 12/30 Flush 13/30 Write block 303 len 16384 data: 8 6 7 12 14/30 Read block 367 len 24576 15/30 Read block 594 len 12288 16/30 Write block 268 len 4096 data: 15 17/30 Read block 44 len 24576 18/30 Write block 376 len 24576 data: 9 12 14 14 11 12 19/30 Read block 294 len 20480 20/30 Read block 47 len 24576 21/30 Read block 321 len 32768 22/30 Read block 100 len 20480 23/30 Write block 183 len 40960 data: 10 10 11 12 12 9 8 7 8 8 24/30 Read block 215 len 16384 25/30 Write block 436 len 12288 data: 9 9 9 26/30 Read block 393 len 40960 27/30 Write block 361 len 24576 data: 12 12 13 11 11 12 28/30 Read block 184 len 36864 29/30 Read block 484 len 8192 30/30 Write block 6 len 28672 data: 7 10 9 12 12 9 9 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:09.784528802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 259 259 259 T T T 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 150 150 150 261 261 261 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 149 149 149 259 259 259 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 149 149 149 259 259 259 F F F 8 160-179 149 149 149 259 259 259 F F F 9 180-199 149 149 149 259 259 259 T T T 10 200-219 134 134 134 229 229 229 F F F 11 220-239 150 150 150 262 262 262 F F F 12 240-259 150 150 150 262 262 262 F F F 13 260-279 148 148 148 257 257 257 T T T 14 280-299 148 148 148 258 258 258 F F F 15 300-319 145 145 145 252 252 252 T T T 16 320-339 148 148 148 258 258 258 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 146 146 146 253 253 253 T T T 19 380-399 146 146 146 253 253 253 T T T 20 400-419 150 150 150 260 260 260 F F F 21 420-439 150 150 150 262 262 262 T T T 22 440-459 148 148 148 257 257 257 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 258 258 258 F F F 26 520-539 145 145 145 252 252 252 F F F 27 540-559 148 148 148 258 258 258 F F F 28 560-579 145 145 145 252 252 252 F F F 29 580-599 146 146 146 255 255 255 F F F Max gen: 150, Max flush: 262 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 263 263 263 F F F 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 150 150 150 261 261 261 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 149 149 149 259 259 259 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 149 149 149 259 259 259 F F F 8 160-179 149 149 149 259 259 259 F F F 9 180-199 151 151 151 263 263 263 F F F 10 200-219 134 134 134 229 229 229 F F F 11 220-239 150 150 150 262 262 262 F F F 12 240-259 150 150 150 262 262 262 F F F 13 260-279 151 151 151 263 263 263 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 151 151 151 263 263 263 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 151 151 151 263 263 263 F F F 19 380-399 151 151 151 263 263 263 F F F 20 400-419 150 150 150 260 260 260 F F F 21 420-439 151 151 151 263 263 263 F F F 22 440-459 148 148 148 257 257 257 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 258 258 258 F F F 26 520-539 145 145 145 252 252 252 F F F 27 540-559 148 148 148 258 258 258 F F F 28 560-579 145 145 145 252 252 252 F F F 29 580-599 146 146 146 255 255 255 F F F Max gen: 151, Max flush: 263 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.058373404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.10084488Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.101448404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.101483885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.101498415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.101514016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790} {"msg":"Crucible 30cc1b2d-7334-45c4-8c05-1dcdc44b1f96 has session id: b4b3b921-0183-431a-ba65-2c56bf959faa","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.101932766Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Upstairs opts: Upstairs UUID: 30cc1b2d-7334-45c4-8c05-1dcdc44b1f96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.101977877Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Crucible stats registered with UUID: 30cc1b2d-7334-45c4-8c05-1dcdc44b1f96","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.102002458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.131781659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.13182265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790} {"msg":"30cc1b2d-7334-45c4-8c05-1dcdc44b1f96 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.131879382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.131913002Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.131955554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.131988164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132016235Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132043486Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132072536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132104267Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132134128Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132164428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:43412)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132196299Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:38765)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.13222794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:34105)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132258681Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132757383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132785083Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132804734Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132819574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132932767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.132957638Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.133393218Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.13348031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.133737176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.133950771Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.133975752Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.133991062Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134030043Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134047024Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134060344Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134227468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134245918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[0]R flush_numbers[0..12]: [263, 255, 240, 261, 259, 259, 247, 259, 259, 263, 229, 262]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134263829Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[0]R generation[0..12]: [151, 146, 138, 150, 149, 149, 142, 149, 149, 151, 134, 150]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134278459Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134292689Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[1]R flush_numbers[0..12]: [263, 255, 240, 261, 259, 259, 247, 259, 259, 263, 229, 262]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.13430665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[1]R generation[0..12]: [151, 146, 138, 150, 149, 149, 142, 149, 149, 151, 134, 150]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.13432038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.13433445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[2]R flush_numbers[0..12]: [263, 255, 240, 261, 259, 259, 247, 259, 259, 263, 229, 262]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134348321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[2]R generation[0..12]: [151, 146, 138, 150, 149, 149, 142, 149, 149, 151, 134, 150]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134362011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134376021Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134389272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134402562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134415652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134428683Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"":"downstairs","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134442123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134466563Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"0","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134480324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"1","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134493774Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"client":"2","":"downstairs","session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134507215Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134519725Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"30cc1b2d-7334-45c4-8c05-1dcdc44b1f96 is now active with session: b4b3b921-0183-431a-ba65-2c56bf959faa","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134532405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134544365Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790,"session_id":"b4b3b921-0183-431a-ba65-2c56bf959faa"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:14.134556776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 207 len 28672 data: 10 11 10 8 9 8 9 02/30 Read block 253 len 12288 03/30 Read block 413 len 28672 04/30 Flush 05/30 Write block 409 len 12288 data: 16 15 15 06/30 Write block 522 len 28672 data: 8 8 7 7 5 6 8 07/30 Write block 586 len 16384 data: 10 11 10 11 08/30 Read block 87 len 4096 09/30 Flush 10/30 Write block 357 len 20480 data: 9 8 10 11 13 11/30 Read block 276 len 32768 12/30 Read block 0 len 16384 13/30 Read block 201 len 12288 14/30 Write block 12 len 28672 data: 10 9 9 10 12 12 10 15/30 Read block 225 len 40960 16/30 Write block 549 len 4096 data: 10 17/30 Read block 195 len 8192 18/30 Read block 92 len 28672 19/30 Write block 444 len 20480 data: 9 8 9 8 8 20/30 Write block 565 len 32768 data: 7 6 7 7 10 11 13 14 21/30 Write block 190 len 32768 data: 8 9 9 11 12 14 13 15 22/30 Read block 48 len 28672 23/30 Write block 205 len 20480 data: 10 10 11 12 11 24/30 Read block 561 len 4096 25/30 Read block 115 len 24576 26/30 Read block 444 len 24576 27/30 Write block 537 len 16384 data: 14 14 14 17 28/30 Read block 488 len 28672 29/30 Read block 86 len 28672 30/30 Write block 499 len 24576 data: 11 12 15 15 17 15 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 5 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:14.154113673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 263 263 263 T T T 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 150 150 150 261 261 261 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 149 149 149 259 259 259 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 149 149 149 259 259 259 F F F 8 160-179 149 149 149 259 259 259 F F F 9 180-199 151 151 151 263 263 263 T T T 10 200-219 152 152 152 264 264 264 T T T 11 220-239 150 150 150 262 262 262 F F F 12 240-259 150 150 150 262 262 262 F F F 13 260-279 151 151 151 263 263 263 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 151 151 151 263 263 263 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 138 138 138 240 240 240 T T T 18 360-379 151 151 151 263 263 263 T T T 19 380-399 151 151 151 263 263 263 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 151 151 151 263 263 263 F F F 22 440-459 148 148 148 257 257 257 T T T 23 460-479 142 142 142 246 246 246 F F F 24 480-499 148 148 148 258 258 258 T T T 25 500-519 148 148 148 258 258 258 T T T 26 520-539 152 152 152 265 265 265 T T T 27 540-559 148 148 148 258 258 258 T T T 28 560-579 145 145 145 252 252 252 T T T 29 580-599 152 152 152 265 265 265 F F F Max gen: 152, Max flush: 265 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 266 266 266 F F F 1 020-039 146 146 146 255 255 255 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 150 150 150 261 261 261 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 149 149 149 259 259 259 F F F 6 120-139 142 142 142 247 247 247 F F F 7 140-159 149 149 149 259 259 259 F F F 8 160-179 149 149 149 259 259 259 F F F 9 180-199 153 153 153 266 266 266 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 150 150 150 262 262 262 F F F 12 240-259 150 150 150 262 262 262 F F F 13 260-279 151 151 151 263 263 263 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 151 151 151 263 263 263 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 153 153 153 266 266 266 F F F 18 360-379 153 153 153 266 266 266 F F F 19 380-399 151 151 151 263 263 263 F F F 20 400-419 152 152 152 265 265 265 F F F 21 420-439 151 151 151 263 263 263 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 153 153 153 266 266 266 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 F F F 27 540-559 153 153 153 266 266 266 F F F 28 560-579 153 153 153 266 266 266 F F F 29 580-599 152 152 152 265 265 265 F F F Max gen: 153, Max flush: 266 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.4520031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.494460855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.495047399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.49508369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.49509873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.49511336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798} {"msg":"Crucible 3277a7b8-26b5-49f6-860a-949875867fb5 has session id: 70b0c9d4-0385-475a-9308-f673ba2b808f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.495593832Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Upstairs opts: Upstairs UUID: 3277a7b8-26b5-49f6-860a-949875867fb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.495639183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Crucible stats registered with UUID: 3277a7b8-26b5-49f6-860a-949875867fb5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.495656933Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.525927937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.525965888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798} {"msg":"3277a7b8-26b5-49f6-860a-949875867fb5 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.52604878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526071251Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526101421Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526119412Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526136592Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526209894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526249135Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526276235Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526303976Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526331697Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58319)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526362127Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40170)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.52647023Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48551)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.526504241Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527091425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527119485Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527141396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527180837Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527198818Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527212608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527681119Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.52773266Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.527845893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528170581Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528197671Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528223622Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528248712Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528274173Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528292113Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528305254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528318924Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[0]R flush_numbers[0..12]: [266, 255, 240, 261, 259, 259, 247, 259, 259, 266, 266, 262]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528346945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[0]R generation[0..12]: [153, 146, 138, 150, 149, 149, 142, 149, 149, 153, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528367155Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528382246Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[1]R flush_numbers[0..12]: [266, 255, 240, 261, 259, 259, 247, 259, 259, 266, 266, 262]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528396406Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[1]R generation[0..12]: [153, 146, 138, 150, 149, 149, 142, 149, 149, 153, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528410196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528424337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[2]R flush_numbers[0..12]: [266, 255, 240, 261, 259, 259, 247, 259, 259, 266, 266, 262]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528438217Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[2]R generation[0..12]: [153, 146, 138, 150, 149, 149, 142, 149, 149, 153, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528452168Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528466108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528479258Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528492568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528505479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:13:18.528518559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"":"downstairs","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.52853675Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.52854914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"0","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.5285649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"1","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528586121Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"client":"2","":"downstairs","session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528608351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528626992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"3277a7b8-26b5-49f6-860a-949875867fb5 is now active with session: 70b0c9d4-0385-475a-9308-f673ba2b808f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528646742Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528664782Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798,"session_id":"70b0c9d4-0385-475a-9308-f673ba2b808f"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:18.528683233Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 411 len 20480 data: 16 15 16 14 14 02/30 Read block 70 len 28672 03/30 Read block 543 len 32768 04/30 Read block 568 len 16384 05/30 Write block 487 len 32768 data: 13 11 11 11 11 10 9 12 06/30 Read block 17 len 12288 07/30 Read block 522 len 28672 08/30 Flush 09/30 Write block 429 len 4096 data: 14 10/30 Read block 88 len 24576 11/30 Write block 168 len 20480 data: 7 6 5 5 6 12/30 Read block 233 len 16384 13/30 Write block 225 len 24576 data: 16 13 12 9 10 9 14/30 Write block 123 len 12288 data: 14 14 15 15/30 Read block 82 len 20480 16/30 Write block 437 len 12288 data: 10 10 12 17/30 Write block 381 len 16384 data: 13 13 11 11 18/30 Write block 268 len 12288 data: 16 19 17 19/30 Write block 402 len 20480 data: 17 19 15 12 11 20/30 Write block 18 len 12288 data: 11 10 12 21/30 Read block 440 len 4096 22/30 Read block 256 len 40960 23/30 Flush 24/30 Write block 122 len 8192 data: 14 15 25/30 Flush 26/30 Write block 571 len 4096 data: 14 27/30 Write block 13 len 8192 data: 10 10 28/30 Read block 427 len 4096 29/30 Read block 13 len 20480 30/30 Read block 439 len 12288 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:18.546837077Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 268 268 268 T T T 1 020-039 154 154 154 268 268 268 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 150 150 150 261 261 261 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 149 149 149 259 259 259 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 149 149 149 259 259 259 F F F 8 160-179 154 154 154 268 268 268 F F F 9 180-199 153 153 153 266 266 266 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 154 154 154 268 268 268 F F F 12 240-259 150 150 150 262 262 262 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 151 151 151 263 263 263 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 153 153 153 266 266 266 F F F 18 360-379 153 153 153 266 266 266 F F F 19 380-399 154 154 154 268 268 268 F F F 20 400-419 154 154 154 268 268 268 F F F 21 420-439 154 154 154 268 268 268 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 154 154 154 267 267 267 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 F F F 27 540-559 153 153 153 266 266 266 F F F 28 560-579 153 153 153 266 266 266 T T T 29 580-599 152 152 152 265 265 265 F F F Max gen: 154, Max flush: 269 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 270 270 270 F F F 1 020-039 154 154 154 268 268 268 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 150 150 150 261 261 261 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 149 149 149 259 259 259 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 149 149 149 259 259 259 F F F 8 160-179 154 154 154 268 268 268 F F F 9 180-199 153 153 153 266 266 266 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 154 154 154 268 268 268 F F F 12 240-259 150 150 150 262 262 262 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 151 151 151 263 263 263 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 153 153 153 266 266 266 F F F 18 360-379 153 153 153 266 266 266 F F F 19 380-399 154 154 154 268 268 268 F F F 20 400-419 154 154 154 268 268 268 F F F 21 420-439 154 154 154 268 268 268 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 154 154 154 267 267 267 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 F F F 27 540-559 153 153 153 266 266 266 F F F 28 560-579 155 155 155 270 270 270 F F F 29 580-599 152 152 152 265 265 265 F F F Max gen: 155, Max flush: 270 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.789232787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.831912858Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.832558873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.832594354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.832609044Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.832623905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806} {"msg":"Crucible 80a683f1-194a-4a7a-9f1d-4478c3e33ee6 has session id: 20c3b3b2-a363-4745-9bdf-55b139568db9","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.833133687Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Upstairs opts: Upstairs UUID: 80a683f1-194a-4a7a-9f1d-4478c3e33ee6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.833178558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Crucible stats registered with UUID: 80a683f1-194a-4a7a-9f1d-4478c3e33ee6","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.833205479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.862790216Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.862836037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806} {"msg":"80a683f1-194a-4a7a-9f1d-4478c3e33ee6 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.862905448Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.862934159Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86296056Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86297602Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863041672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863082463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863109364Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863138194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863166995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863196326Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62063)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863324449Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63232)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863362639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46058)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86339193Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863959804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.863988354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.864140028Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.864180329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86420928Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86423649Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.864606999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86465286Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.864963418Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865182423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865209134Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865223764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865252225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865266885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865280965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865407798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865426739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[0]R flush_numbers[0..12]: [270, 268, 240, 261, 259, 259, 269, 259, 268, 266, 266, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865441539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[0]R generation[0..12]: [155, 154, 138, 150, 149, 149, 154, 149, 154, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.8654557Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86547031Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[1]R flush_numbers[0..12]: [270, 268, 240, 261, 259, 259, 269, 259, 268, 266, 266, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.86548446Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[1]R generation[0..12]: [155, 154, 138, 150, 149, 149, 154, 149, 154, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865502351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865517791Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[2]R flush_numbers[0..12]: [270, 268, 240, 261, 259, 259, 269, 259, 268, 266, 266, 268]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865532041Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[2]R generation[0..12]: [155, 154, 138, 150, 149, 149, 154, 149, 154, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865545992Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865560652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865574082Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865587823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865600953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865621693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"":"downstairs","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865635504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865647834Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"0","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865661345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"1","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865674715Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"client":"2","":"downstairs","session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865688315Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865700915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"80a683f1-194a-4a7a-9f1d-4478c3e33ee6 is now active with session: 20c3b3b2-a363-4745-9bdf-55b139568db9","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865715856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865728396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806,"session_id":"20c3b3b2-a363-4745-9bdf-55b139568db9"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:22.865740966Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 335 len 24576 data: 13 9 10 12 11 12 03/30 Read block 252 len 12288 04/30 Read block 299 len 4096 05/30 Write block 237 len 24576 data: 13 15 13 12 15 15 06/30 Write block 59 len 28672 data: 14 13 15 14 13 15 18 07/30 Read block 280 len 8192 08/30 Write block 383 len 20480 data: 12 12 10 11 14 09/30 Write block 377 len 12288 data: 13 15 15 10/30 Read block 46 len 16384 11/30 Write block 146 len 24576 data: 10 11 9 9 9 9 12/30 Write block 426 len 36864 data: 15 15 16 15 15 12 11 10 12 13/30 Read block 109 len 4096 14/30 Write block 5 len 24576 data: 8 8 11 10 13 13 15/30 Flush 16/30 Flush 17/30 Write block 103 len 4096 data: 12 18/30 Read block 423 len 36864 19/30 Read block 224 len 20480 20/30 Read block 247 len 8192 21/30 Read block 400 len 12288 22/30 Flush 23/30 Write block 588 len 28672 data: 11 12 11 11 11 10 10 24/30 Read block 63 len 40960 25/30 Write block 325 len 20480 data: 13 14 13 12 11 26/30 Write block 2 len 32768 data: 5 7 9 9 9 12 11 14 27/30 Write block 290 len 40960 data: 15 17 20 15 15 16 16 18 13 8 28/30 Write block 180 len 4096 data: 13 29/30 Write block 357 len 24576 data: 10 9 11 12 14 13 30/30 Read block 454 len 40960 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:22.884372712Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 T T T 1 020-039 154 154 154 268 268 268 F F F 2 040-059 156 156 156 272 272 272 F F F 3 060-079 156 156 156 272 272 272 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 156 156 156 272 272 272 F F F 8 160-179 154 154 154 268 268 268 F F F 9 180-199 153 153 153 266 266 266 T T T 10 200-219 153 153 153 266 266 266 F F F 11 220-239 156 156 156 272 272 272 F F F 12 240-259 156 156 156 272 272 272 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 148 148 148 258 258 258 T T T 15 300-319 151 151 151 263 263 263 F F F 16 320-339 156 156 156 272 272 272 T T T 17 340-359 156 156 156 272 272 272 T T T 18 360-379 156 156 156 272 272 272 T T T 19 380-399 156 156 156 272 272 272 F F F 20 400-419 154 154 154 268 268 268 F F F 21 420-439 156 156 156 272 272 272 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 154 154 154 267 267 267 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 F F F 27 540-559 153 153 153 266 266 266 F F F 28 560-579 155 155 155 270 270 270 F F F 29 580-599 152 152 152 265 265 265 T T T Max gen: 156, Max flush: 274 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 154 154 154 268 268 268 F F F 2 040-059 156 156 156 272 272 272 F F F 3 060-079 156 156 156 272 272 272 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 156 156 156 272 272 272 F F F 8 160-179 154 154 154 268 268 268 F F F 9 180-199 157 157 157 275 275 275 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 156 156 156 272 272 272 F F F 12 240-259 156 156 156 272 272 272 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 157 157 157 275 275 275 F F F 15 300-319 151 151 151 263 263 263 F F F 16 320-339 157 157 157 275 275 275 F F F 17 340-359 157 157 157 275 275 275 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 156 156 156 272 272 272 F F F 20 400-419 154 154 154 268 268 268 F F F 21 420-439 156 156 156 272 272 272 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 154 154 154 267 267 267 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 F F F 27 540-559 153 153 153 266 266 266 F F F 28 560-579 155 155 155 270 270 270 F F F 29 580-599 157 157 157 275 275 275 F F F Max gen: 157, Max flush: 275 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.159191758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:27.201285544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.201640952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.2329063Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.233468103Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.233503914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.233518194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.233533575Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814} {"msg":"Crucible 1bb5286f-f03f-4124-94d9-ca89a2776380 has session id: 21aaf645-6a27-4f56-ab18-c691b62bcf12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.233971945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Upstairs opts: Upstairs UUID: 1bb5286f-f03f-4124-94d9-ca89a2776380, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.234012806Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Crucible stats registered with UUID: 1bb5286f-f03f-4124-94d9-ca89a2776380","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.234030036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264386222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264417273Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814} {"msg":"1bb5286f-f03f-4124-94d9-ca89a2776380 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264495255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264516605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264545606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264563926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264581567Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264603027Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264655729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264756131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264799512Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264836033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_connection connected from Ok(127.0.0.1:57396)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264945535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_connection connected from Ok(127.0.0.1:52604)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.264990177Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_connection connected from Ok(127.0.0.1:64849)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.265030148Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.265475478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.265513019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.26553459Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.26555917Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.265597281Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.265632372Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266077113Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266224816Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266361109Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266642226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266667837Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266683767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266724988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266742638Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266756739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266831481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266850551Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[0]R flush_numbers[0..12]: [275, 268, 272, 272, 259, 274, 269, 272, 268, 275, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266865061Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[0]R generation[0..12]: [157, 154, 156, 156, 149, 156, 154, 156, 154, 157, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266879182Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266893702Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[1]R flush_numbers[0..12]: [275, 268, 272, 272, 259, 274, 269, 272, 268, 275, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266907862Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[1]R generation[0..12]: [157, 154, 156, 156, 149, 156, 154, 156, 154, 157, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266921873Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266940093Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[2]R flush_numbers[0..12]: [275, 268, 272, 272, 259, 274, 269, 272, 268, 275, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266955054Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[2]R generation[0..12]: [157, 154, 156, 156, 149, 156, 154, 156, 154, 157, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266969294Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.266983654Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267000415Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267024185Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267047526Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267061026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"":"downstairs","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267074537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267086897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"0","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267100297Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"1","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267113847Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"client":"2","":"downstairs","session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267127548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267140308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"1bb5286f-f03f-4124-94d9-ca89a2776380 is now active with session: 21aaf645-6a27-4f56-ab18-c691b62bcf12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267153448Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,"session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267168079Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814,Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "session_id":"21aaf645-6a27-4f56-ab18-c691b62bcf12"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:27.267185099Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 575 len 36864 data: 16 14 13 14 13 13 12 10 11 02/30 Read block 317 len 12288 03/30 Read block 568 len 36864 04/30 Read block 151 len 28672 05/30 Read block 522 len 16384 06/30 Write block 419 len 24576 data: 11 11 13 15 14 13 07/30 Read block 177 len 20480 08/30 Read block 428 len 24576 09/30 Write block 552 len 8192 data: 16 13 10/30 Read block 10 len 12288 11/30 Write block 259 len 4096 data: 10 12/30 Read block 478 len 12288 13/30 Read block 172 len 12288 14/30 Flush 15/30 Write block 32 len 28672 data: 12 13 14 12 16 16 17 16/30 Flush 17/30 Write block 306 len 36864 data: 13 11 10 9 11 11 10 13 15 18/30 Write block 283 len 8192 data: 11 13 19/30 Read block 465 len 32768 20/30 Write block 485 len 4096 data: 13 21/30 Write block 394 len 12288 data: 12 12 13 22/30 Read block 205 len 8192 23/30 Read block 80 len 36864 24/30 Write block 573 len 8192 data: 13 15 25/30 Read block 506 len 32768 26/30 Read block 224 len 8192 27/30 Write block 76 len 16384 data: 6 10 9 7 28/30 Write block 147 len 16384 data: 12 10 10 10 29/30 Read block 484 len 4096 30/30 Write block 582 len 40960 data: 11 12 10 8 11 12 12 13 12 12 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 9 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:27.286200144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 158 158 158 277 277 277 F F F 2 040-059 156 156 156 272 272 272 F F F 3 060-079 156 156 156 272 272 272 T T T 4 080-099 149 149 149 259 259 259 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 156 156 156 272 272 272 T T T 8 160-179 154 154 154 268 268 268 F F F 9 180-199 157 157 157 275 275 275 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 156 156 156 272 272 272 F F F 12 240-259 158 158 158 276 276 276 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 157 157 157 275 275 275 T T T 15 300-319 151 151 151 263 263 263 T T T 16 320-339 157 157 157 275 275 275 F F F 17 340-359 157 157 157 275 275 275 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 156 156 156 272 272 272 T T T 20 400-419 158 158 158 276 276 276 F F F 21 420-439 158 158 158 276 276 276 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 154 154 154 267 267 267 T T T 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 F F F 27 540-559 158 158 158 276 276 276 F F F 28 560-579 158 158 158 276 276 276 T T T 29 580-599 158 158 158 276 276 276 T T T Max gen: 158, Max flush: 277 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 158 158 158 277 277 277 F F F 2 040-059 156 156 156 272 272 272 F F F 3 060-079 159 159 159 278 278 278 F F F 4 080-099 149 149 149 259 259 259 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 159 159 159 278 278 278 F F F 8 160-179 154 154 154 268 268 268 F F F 9 180-199 157 157 157 275 275 275 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 156 156 156 272 272 272 F F F 12 240-259 158 158 158 276 276 276 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 159 159 159 278 278 278 F F F 15 300-319 159 159 159 278 278 278 F F F 16 320-339 157 157 157 275 275 275 F F F 17 340-359 157 157 157 275 275 275 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 159 159 159 278 278 278 F F F 20 400-419 158 158 158 276 276 276 F F F 21 420-439 158 158 158 276 276 276 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 F F F 27 540-559 158 158 158 276 276 276 F F F 28 560-579 159 159 159 278 278 278 F F F 29 580-599 159 159 159 278 278 278 F F F Max gen: 159, Max flush: 278 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.596929207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:31.638304216Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.638698736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.669046051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.669615525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.669651256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.669665666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.669680237Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822} {"msg":"Crucible 08c73651-07f8-4386-9af9-b4a0c9bbda95 has session id: cedd3021-1bd1-4247-af31-a7ed583779ef","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.670055315Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Upstairs opts: Upstairs UUID: 08c73651-07f8-4386-9af9-b4a0c9bbda95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.670092636Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Crucible stats registered with UUID: 08c73651-07f8-4386-9af9-b4a0c9bbda95","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.670109467Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.6995558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699587371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822} {"msg":"08c73651-07f8-4386-9af9-b4a0c9bbda95 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699620222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699636593Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699666593Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699685284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699702314Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699722344Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699760025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699793736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.699830957Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700054743Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:46086)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700089953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:35224)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700109284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:58336)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700127274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700717298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700758409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.70079081Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700818841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700848291Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.700878502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.701369144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.701440656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.701682851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.701931097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.701971348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702002469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.70203248Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.70205224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702067131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702175123Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702195154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[0]R flush_numbers[0..12]: [275, 277, 272, 278, 259, 274, 269, 278, 268, 275, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702209574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[0]R generation[0..12]: [157, 158, 156, 159, 149, 156, 154, 159, 154, 157, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702223794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702237995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[1]R flush_numbers[0..12]: [275, 277, 272, 278, 259, 274, 269, 278, 268, 275, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702251845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[1]R generation[0..12]: [157, 158, 156, 159, 149, 156, 154, 159, 154, 157, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702265525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702282756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[2]R flush_numbers[0..12]: [275, 277, 272, 278, 259, 274, 269, 278, 268, 275, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702297146Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[2]R generation[0..12]: [157, 158, 156, 159, 149, 156, 154, 159, 154, 157, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702311197Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702325227Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702338347Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702351667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702364798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:13:31.702377888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"":"downstairs","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702403449Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702428719Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"0","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.7024539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"1","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702479861Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"client":"2","":"downstairs","session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702505511Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702532772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"08c73651-07f8-4386-9af9-b4a0c9bbda95 is now active with session: cedd3021-1bd1-4247-af31-a7ed583779ef","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702552392Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702567923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822,"session_id":"cedd3021-1bd1-4247-af31-a7ed583779ef"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:31.702580573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 33 len 20480 data: 14 15 13 17 17 02/30 Read block 251 len 8192 03/30 Flush 04/30 Write block 180 len 4096 data: 14 05/30 Read block 97 len 36864 06/30 Read block 412 len 20480 07/30 Write block 44 len 28672 data: 15 15 16 14 12 15 19 08/30 Write block 28 len 28672 data: 13 11 14 15 13 15 16 09/30 Read block 531 len 8192 10/30 Read block 415 len 16384 11/30 Write block 584 len 32768 data: 11 9 12 13 13 14 13 13 12/30 Flush 13/30 Flush 14/30 Write block 77 len 24576 data: 11 10 8 6 10 8 15/30 Flush 16/30 Write block 319 len 16384 data: 14 13 14 11 17/30 Flush 18/30 Write block 35 len 28672 data: 14 18 18 18 15 13 14 19/30 Write block 241 len 36864 data: 16 16 13 11 11 8 7 5 6 20/30 Read block 377 len 4096 21/30 Read block 466 len 28672 22/30 Read block 512 len 24576 23/30 Flush 24/30 Write block 537 len 32768 data: 15 15 15 18 14 15 15 15 25/30 Read block 573 len 8192 26/30 Read block 205 len 4096 27/30 Read block 260 len 12288 28/30 Read block 206 len 8192 29/30 Read block 507 len 16384 30/30 Read block 393 len 8192 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:31.722038718Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 160 160 160 284 284 284 F F F 3 060-079 160 160 160 282 282 282 F F F 4 080-099 160 160 160 282 282 282 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 159 159 159 278 278 278 F F F 8 160-179 154 154 154 268 268 268 F F F 9 180-199 160 160 160 280 280 280 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 156 156 156 272 272 272 F F F 12 240-259 160 160 160 284 284 284 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 159 159 159 278 278 278 F F F 15 300-319 160 160 160 283 283 283 F F F 16 320-339 160 160 160 283 283 283 F F F 17 340-359 157 157 157 275 275 275 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 159 159 159 278 278 278 F F F 20 400-419 158 158 158 276 276 276 F F F 21 420-439 158 158 158 276 276 276 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 153 153 153 266 266 266 T T T 27 540-559 158 158 158 276 276 276 T T T 28 560-579 159 159 159 278 278 278 F F F 29 580-599 160 160 160 280 280 280 F F F Max gen: 160, Max flush: 284 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 160 160 160 284 284 284 F F F 3 060-079 160 160 160 282 282 282 F F F 4 080-099 160 160 160 282 282 282 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 154 154 154 269 269 269 F F F 7 140-159 159 159 159 278 278 278 F F F 8 160-179 154 154 154 268 268 268 F F F 9 180-199 160 160 160 280 280 280 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 156 156 156 272 272 272 F F F 12 240-259 160 160 160 284 284 284 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 159 159 159 278 278 278 F F F 15 300-319 160 160 160 283 283 283 F F F 16 320-339 160 160 160 283 283 283 F F F 17 340-359 157 157 157 275 275 275 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 159 159 159 278 278 278 F F F 20 400-419 158 158 158 276 276 276 F F F 21 420-439 158 158 158 276 276 276 F F F 22 440-459 153 153 153 266 266 266 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 159 159 159 278 278 278 F F F 29 580-599 160 160 160 280 280 280 F F F Max gen: 161, Max flush: 285 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:35.999377673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.043101988Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.043718823Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.043755454Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.043770395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.043784995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830} {"msg":"Crucible 83cc3564-b595-468c-94ad-41ec8a5e3de3 has session id: 58f33376-bedf-4342-be08-e5b224cf445c","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.044199815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Upstairs opts: Upstairs UUID: 83cc3564-b595-468c-94ad-41ec8a5e3de3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.044245316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Crucible stats registered with UUID: 83cc3564-b595-468c-94ad-41ec8a5e3de3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.044270216Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074024908Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074051598Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830} {"msg":"83cc3564-b595-468c-94ad-41ec8a5e3de3 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.07413089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074153501Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074193222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074224423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074256833Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074284044Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074306894Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074321875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074335935Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074353815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55134)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074374986Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51519)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074402437Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50443)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.074431377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075127044Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075169135Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075204286Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075235397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075261797Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075290238Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.07578149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075944513Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.075976284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076321703Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076352973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076381914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076513387Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076537688Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076552048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076577829Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076592149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[0]R flush_numbers[0..12]: [275, 284, 284, 282, 282, 274, 269, 278, 268, 280, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.07663153Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[0]R generation[0..12]: [157, 160, 160, 160, 160, 156, 154, 159, 154, 160, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.07664987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076664311Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[1]R flush_numbers[0..12]: [275, 284, 284, 282, 282, 274, 269, 278, 268, 280, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076678521Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[1]R generation[0..12]: [157, 160, 160, 160, 160, 156, 154, 159, 154, 160, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076692411Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076706652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[2]R flush_numbers[0..12]: [275, 284, 284, 282, 282, 274, 269, 278, 268, 280, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076720572Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[2]R generation[0..12]: [157, 160, 160, 160, 160, 156, 154, 159, 154, 160, 153, 156]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076734563Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076748623Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076765143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076779164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:13:36.076792284Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076812664Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"":"downstairs","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076826035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076838355Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"0","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076851815Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"1","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076865246Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"client":"2","":"downstairs","session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076878596Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076890946Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"83cc3564-b595-468c-94ad-41ec8a5e3de3 is now active with session: 58f33376-bedf-4342-be08-e5b224cf445c","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076903867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076915947Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830,"session_id":"58f33376-bedf-4342-be08-e5b224cf445c"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:36.076929147Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 291 len 32768 data: 18 21 16 16 17 17 19 14 02/30 Read block 27 len 4096 03/30 Write block 225 len 36864 data: 17 14 13 10 11 10 8 10 13 04/30 Write block 325 len 4096 data: 14 05/30 Read block 466 len 8192 06/30 Read block 35 len 32768 07/30 Read block 440 len 24576 08/30 Flush 09/30 Write block 423 len 16384 data: 15 14 15 16 10/30 Read block 487 len 12288 11/30 Write block 178 len 20480 data: 11 11 15 10 10 12/30 Read block 499 len 40960 13/30 Flush 14/30 Write block 410 len 28672 data: 16 17 16 17 15 15 13 15/30 Write block 184 len 4096 data: 11 16/30 Write block 435 len 20480 data: 11 10 11 11 13 17/30 Write block 444 len 12288 data: 10 9 10 18/30 Write block 138 len 16384 data: 8 10 14 17 19/30 Read block 158 len 40960 20/30 Write block 48 len 4096 data: 13 21/30 Read block 465 len 24576 22/30 Flush 23/30 Write block 574 len 28672 data: 16 17 15 14 15 14 14 24/30 Flush 25/30 Write block 90 len 36864 data: 13 13 14 16 12 14 14 12 14 26/30 Read block 421 len 36864 27/30 Read block 0 len 24576 28/30 Read block 86 len 36864 29/30 Read block 507 len 24576 30/30 Read block 100 len 8192 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:36.096398973Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 160 160 160 282 282 282 F F F 4 080-099 160 160 160 282 282 282 T T T 5 100-119 156 156 156 274 274 274 F F F 6 120-139 162 162 162 288 288 288 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 162 162 162 288 288 288 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 162 162 162 286 286 286 F F F 12 240-259 160 160 160 284 284 284 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 162 162 162 286 286 286 F F F 15 300-319 160 160 160 283 283 283 F F F 16 320-339 162 162 162 286 286 286 F F F 17 340-359 157 157 157 275 275 275 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 159 159 159 278 278 278 F F F 20 400-419 162 162 162 288 288 288 F F F 21 420-439 162 162 162 288 288 288 F F F 22 440-459 162 162 162 288 288 288 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 162 162 162 289 289 289 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 162, Max flush: 289 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 160 160 160 282 282 282 F F F 4 080-099 163 163 163 290 290 290 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 162 162 162 288 288 288 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 162 162 162 288 288 288 F F F 10 200-219 153 153 153 266 266 266 F F F 11 220-239 162 162 162 286 286 286 F F F 12 240-259 160 160 160 284 284 284 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 162 162 162 286 286 286 F F F 15 300-319 160 160 160 283 283 283 F F F 16 320-339 162 162 162 286 286 286 F F F 17 340-359 157 157 157 275 275 275 F F F 18 360-379 157 157 157 275 275 275 F F F 19 380-399 159 159 159 278 278 278 F F F 20 400-419 162 162 162 288 288 288 F F F 21 420-439 162 162 162 288 288 288 F F F 22 440-459 162 162 162 288 288 288 F F F 23 460-479 142 142 142 246 246 246 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 162 162 162 289 289 289 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 163, Max flush: 290 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.334632693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:40.375947421Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.376357841Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.406660395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.407225468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.407260269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.40727506Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.40728985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838} {"msg":"Crucible c0db77f9-5252-457e-a10b-e2caaf186c63 has session id: ccecacae-9b56-4458-9d54-880794860dd8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.407780902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Upstairs opts: Upstairs UUID: c0db77f9-5252-457e-a10b-e2caaf186c63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.407821383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Crucible stats registered with UUID: c0db77f9-5252-457e-a10b-e2caaf186c63","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.407838433Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437291027Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437321248Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838} {"msg":"c0db77f9-5252-457e-a10b-e2caaf186c63 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437354099Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437370759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.4374001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.43741926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437436751Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437453701Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437470802Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437487492Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437517443Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437551553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38989)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437585744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49812)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437627875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_connection connected from Ok(127.0.0.1:57017)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.437666516Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.438400304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.438429445Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.438448585Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.438474755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.438502686Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.438530257Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.43909085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439122191Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439153322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439633423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439661474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439676124Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439743306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439763466Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439781577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439794517Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439807958Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[0]R flush_numbers[0..12]: [275, 284, 288, 282, 290, 274, 288, 288, 287, 288, 266, 286]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439821918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[0]R generation[0..12]: [157, 160, 162, 160, 163, 156, 162, 162, 162, 162, 153, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439836458Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439850808Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[1]R flush_numbers[0..12]: [275, 284, 288, 282, 290, 274, 288, 288, 287, 288, 266, 286]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439864739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[1]R generation[0..12]: [157, 160, 162, 160, 163, 156, 162, 162, 162, 162, 153, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439878489Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439892589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[2]R flush_numbers[0..12]: [275, 284, 288, 282, 290, 274, 288, 288, 287, 288, 266, 286]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.43990643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[2]R generation[0..12]: [157, 160, 162, 160, 163, 156, 162, 162, 162, 162, 153, 162]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.43992026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439934381Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439947601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439960711Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.439976211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"All extents match","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-12-09T15:13:40.440041963Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"":"downstairs","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440063533Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440076074Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"0","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440089504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"1","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440102645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"client":"2","":"downstairs","session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440118875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440135855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"c0db77f9-5252-457e-a10b-e2caaf186c63 is now active with session: ccecacae-9b56-4458-9d54-880794860dd8","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440155936Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440180126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838,"session_id":"ccecacae-9b56-4458-9d54-880794860dd8"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:40.440204817Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 20480 data: 12 13 12 14 15 02/30 Read block 434 len 40960 03/30 Write block 82 len 40960 data: 9 8 8 6 4 4 9 12 14 14 04/30 Write block 450 len 12288 data: 11 11 10 05/30 Read block 238 len 20480 06/30 Write block 354 len 16384 data: 13 10 10 11 07/30 Write block 299 len 8192 data: 9 7 08/30 Read block 213 len 4096 09/30 Read block 423 len 20480 10/30 Read block 250 len 32768 11/30 Write block 292 len 40960 data: 22 17 17 18 18 20 15 10 8 7 12/30 Read block 432 len 24576 13/30 Flush 14/30 Write block 313 len 24576 data: 14 16 17 18 16 16 15/30 Read block 500 len 24576 16/30 Write block 475 len 20480 data: 5 6 6 9 12 17/30 Read block 474 len 16384 18/30 Write block 71 len 8192 data: 10 8 19/30 Write block 375 len 4096 data: 7 20/30 Write block 242 len 40960 data: 17 14 12 12 9 8 6 7 7 7 21/30 Write block 196 len 40960 data: 14 16 14 14 12 13 11 12 8 11 22/30 Read block 235 len 40960 23/30 Flush 24/30 Write block 469 len 4096 data: 9 25/30 Read block 119 len 8192 26/30 Read block 394 len 40960 27/30 Read block 342 len 8192 28/30 Write block 230 len 20480 data: 11 9 11 14 11 29/30 Read block 443 len 20480 30/30 Write block 400 len 8192 data: 11 13 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:40.460121043Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 164 164 164 292 292 292 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 162 162 162 288 288 288 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 164 164 164 292 292 292 F F F 10 200-219 164 164 164 292 292 292 F F F 11 220-239 162 162 162 286 286 286 T T T 12 240-259 164 164 164 292 292 292 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 164 164 164 292 292 292 F F F 16 320-339 162 162 162 286 286 286 F F F 17 340-359 164 164 164 291 291 291 F F F 18 360-379 164 164 164 292 292 292 F F F 19 380-399 159 159 159 278 278 278 F F F 20 400-419 162 162 162 288 288 288 T T T 21 420-439 162 162 162 288 288 288 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 164 164 164 292 292 292 T T T 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 162 162 162 289 289 289 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 164, Max flush: 292 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 164 164 164 292 292 292 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 162 162 162 288 288 288 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 164 164 164 292 292 292 F F F 10 200-219 164 164 164 292 292 292 F F F 11 220-239 165 165 165 293 293 293 F F F 12 240-259 164 164 164 292 292 292 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 164 164 164 292 292 292 F F F 16 320-339 162 162 162 286 286 286 F F F 17 340-359 164 164 164 291 291 291 F F F 18 360-379 164 164 164 292 292 292 F F F 19 380-399 159 159 159 278 278 278 F F F 20 400-419 165 165 165 293 293 293 F F F 21 420-439 162 162 162 288 288 288 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 165 165 165 293 293 293 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 162 162 162 289 289 289 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 165, Max flush: 293 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.737863788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:44.779100493Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.779458942Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.809964201Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.810532065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.810565536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.810579726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.810594906Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846} {"msg":"Crucible 9fc8bfb2-686e-4c90-9614-1c0a402cff87 has session id: d91f9b59-fcc0-4e0c-b795-874b0128f4db","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.811040267Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Upstairs opts: Upstairs UUID: 9fc8bfb2-686e-4c90-9614-1c0a402cff87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.811088548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Crucible stats registered with UUID: 9fc8bfb2-686e-4c90-9614-1c0a402cff87","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.811110989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.84127768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841321361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846} {"msg":"9fc8bfb2-686e-4c90-9614-1c0a402cff87 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841383792Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841413993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841455874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841487345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841518276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841547886Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841571787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841586767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841601118Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841615088Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_connection connected from Ok(127.0.0.1:48514)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841639768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_connection connected from Ok(127.0.0.1:60210)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841667879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_connection connected from Ok(127.0.0.1:53612)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.841833553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.842424817Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.842455018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.842471898Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.8425324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.842566981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.842596231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843039272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843204026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843244657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843572215Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843623916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843640626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843766019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.84379606Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.84381192Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843828941Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843843601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[0]R flush_numbers[0..12]: [275, 284, 288, 292, 291, 274, 288, 288, 287, 292, 292, 293]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843858032Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[0]R generation[0..12]: [157, 160, 162, 164, 164, 156, 162, 162, 162, 164, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843872232Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843886532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[1]R flush_numbers[0..12]: [275, 284, 288, 292, 291, 274, 288, 288, 287, 292, 292, 293]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843900462Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[1]R generation[0..12]: [157, 160, 162, 164, 164, 156, 162, 162, 162, 164, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843914643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843928883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[2]R flush_numbers[0..12]: [275, 284, 288, 292, 291, 274, 288, 288, 287, 292, 292, 293]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843942783Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[2]R generation[0..12]: [157, 160, 162, 164, 164, 156, 162, 162, 162, 164, 164, 165]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843956684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO ":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843979104Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.843992425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844005965Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844019265Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844034536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"":"downstairs","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844048666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844060796Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"0","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844074477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"1","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844087977Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"client":"2","":"downstairs","session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844101417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844113867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"9fc8bfb2-686e-4c90-9614-1c0a402cff87 is now active with session: d91f9b59-fcc0-4e0c-b795-874b0128f4db","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844126948Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844153748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846,"session_id":"d91f9b59-fcc0-4e0c-b795-874b0128f4db"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:44.844167279Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 8192 data: 8 7 02/30 Read block 451 len 20480 03/30 Read block 560 len 32768 04/30 Read block 56 len 28672 05/30 Write block 326 len 32768 data: 15 14 13 12 10 9 12 13 06/30 Read block 160 len 24576 07/30 Read block 523 len 8192 08/30 Read block 437 len 12288 09/30 Read block 571 len 16384 10/30 Read block 563 len 40960 11/30 Read block 442 len 32768 12/30 Write block 382 len 8192 data: 14 13 13/30 Read block 269 len 16384 14/30 Write block 226 len 4096 data: 15 15/30 Read block 534 len 8192 16/30 Write block 71 len 32768 data: 11 9 10 9 7 7 12 11 17/30 Read block 438 len 36864 18/30 Write block 515 len 16384 data: 15 13 12 10 19/30 Write block 384 len 4096 data: 13 20/30 Write block 378 len 32768 data: 16 16 12 14 15 14 14 11 21/30 Read block 554 len 24576 22/30 Read block 250 len 40960 23/30 Write block 225 len 16384 data: 18 16 14 11 24/30 Read block 428 len 16384 25/30 Write block 317 len 40960 data: 17 17 15 14 15 12 13 13 15 16 26/30 Write block 318 len 4096 data: 18 27/30 Write block 347 len 20480 data: 18 16 16 14 13 28/30 Read block 24 len 16384 29/30 Write block 345 len 40960 data: 13 17 19 17 17 15 14 13 12 14 30/30 Write block 572 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:44.862649181Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 164 164 164 292 292 292 T T T 4 080-099 164 164 164 291 291 291 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 162 162 162 288 288 288 T T T 7 140-159 162 162 162 288 288 288 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 164 164 164 292 292 292 F F F 10 200-219 164 164 164 292 292 292 F F F 11 220-239 165 165 165 293 293 293 T T T 12 240-259 164 164 164 292 292 292 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 164 164 164 292 292 292 T T T 16 320-339 162 162 162 286 286 286 T T T 17 340-359 164 164 164 291 291 291 T T T 18 360-379 164 164 164 292 292 292 T T T 19 380-399 159 159 159 278 278 278 T T T 20 400-419 165 165 165 293 293 293 F F F 21 420-439 162 162 162 288 288 288 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 165 165 165 293 293 293 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 153 153 153 266 266 266 T T T 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 162 162 162 289 289 289 T T T 29 580-599 162 162 162 289 289 289 F F F Max gen: 165, Max flush: 293 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 160 160 160 284 284 284 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 166 166 166 294 294 294 F F F 4 080-099 164 164 164 291 291 291 F F F 5 100-119 156 156 156 274 274 274 F F F 6 120-139 166 166 166 294 294 294 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 164 164 164 292 292 292 F F F 10 200-219 164 164 164 292 292 292 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 164 164 164 292 292 292 F F F 13 260-279 154 154 154 268 268 268 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 166 166 166 294 294 294 F F F 16 320-339 166 166 166 294 294 294 F F F 17 340-359 166 166 166 294 294 294 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 166 166 166 294 294 294 F F F 20 400-419 165 165 165 293 293 293 F F F 21 420-439 162 162 162 288 288 288 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 165 165 165 293 293 293 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 166 166 166 294 294 294 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 166 166 166 294 294 294 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 166, Max flush: 294 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.181478278Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.224207029Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.224869995Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.224905176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.224920006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.224935207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854} {"msg":"Crucible cf0fd349-cc3f-488b-9f3d-f8e8a0a9f246 has session id: 16b74ed0-4c83-4c9a-89ed-d014b26dd646","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.225386728Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Upstairs opts: Upstairs UUID: cf0fd349-cc3f-488b-9f3d-f8e8a0a9f246, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.225435939Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Crucible stats registered with UUID: cf0fd349-cc3f-488b-9f3d-f8e8a0a9f246","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.22546195Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255086788Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255121708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854} {"msg":"cf0fd349-cc3f-488b-9f3d-f8e8a0a9f246 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255148659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.25516617Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.25519601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255214401Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255232051Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255249302Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255279772Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255310663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255343444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255376665Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_connection connected from Ok(127.0.0.1:52630)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255411026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_connection connected from Ok(127.0.0.1:55249)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255455047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_connection connected from Ok(127.0.0.1:35594)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.255497048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256193674Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256230975Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256261796Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256323697Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256354008Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256370368Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256781118Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256879341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.256902931Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257395493Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257425614Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257441664Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257519216Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257538726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257556217Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257569337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257583037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[0]R flush_numbers[0..12]: [275, 284, 288, 294, 291, 274, 294, 288, 287, 292, 292, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257597118Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[0]R generation[0..12]: [157, 160, 162, 166, 164, 156, 166, 162, 162, 164, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257611858Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257626579Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[1]R flush_numbers[0..12]: [275, 284, 288, 294, 291, 274, 294, 288, 287, 292, 292, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257640989Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[1]R generation[0..12]: [157, 160, 162, 166, 164, 156, 166, 162, 162, 164, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257655089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257669269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[2]R flush_numbers[0..12]: [275, 284, 288, 294, 291, 274, 294, 288, 287, 292, 292, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.25768336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[2]R generation[0..12]: [157, 160, 162, 166, 164, 156, 166, 162, 162, 164, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.25769745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257711801Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Max found gen is 167","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257725091Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Generation requested: 168 >= found:167","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:13:49.257738381Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257762142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257775362Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"":"downstairs","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257788872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257801143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"0","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257815043Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"1","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257828693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"client":"2","":"downstairs","session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257842244Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257854914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"cf0fd349-cc3f-488b-9f3d-f8e8a0a9f246 is now active with session: 16b74ed0-4c83-4c9a-89ed-d014b26dd646","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257867854Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257880154Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854,"session_id":"16b74ed0-4c83-4c9a-89ed-d014b26dd646"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:49.257892645Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 32 len 12288 data: 14 16 17 02/30 Read block 585 len 32768 03/30 Read block 596 len 16384 04/30 Read block 461 len 24576 05/30 Read block 559 len 4096 06/30 Read block 391 len 4096 07/30 Read block 395 len 28672 08/30 Flush 09/30 Write block 502 len 40960 data: 16 18 16 13 15 16 13 12 13 12 10/30 Flush 11/30 Write block 256 len 40960 data: 10 10 11 11 13 11 11 11 10 10 12/30 Read block 408 len 40960 13/30 Write block 172 len 32768 data: 7 12 12 11 12 9 12 12 14/30 Read block 24 len 32768 15/30 Read block 54 len 24576 16/30 Read block 387 len 40960 17/30 Write block 331 len 8192 data: 10 13 18/30 Flush 19/30 Write block 426 len 16384 data: 17 16 17 16 20/30 Write block 85 len 12288 data: 7 5 5 21/30 Read block 17 len 8192 22/30 Read block 338 len 24576 23/30 Read block 394 len 4096 24/30 Read block 320 len 8192 25/30 Flush 26/30 Write block 99 len 16384 data: 17 19 17 14 27/30 Read block 33 len 20480 28/30 Read block 427 len 8192 29/30 Read block 274 len 32768 30/30 Write block 197 len 16384 data: 17 15 15 13 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:49.27821108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 168 168 168 295 295 295 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 166 166 166 294 294 294 F F F 4 080-099 168 168 168 298 298 298 T T T 5 100-119 156 156 156 274 274 274 T T T 6 120-139 166 166 166 294 294 294 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 168 168 168 297 297 297 F F F 9 180-199 164 164 164 292 292 292 T T T 10 200-219 164 164 164 292 292 292 T T T 11 220-239 166 166 166 294 294 294 F F F 12 240-259 168 168 168 297 297 297 F F F 13 260-279 168 168 168 297 297 297 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 166 166 166 294 294 294 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 166 166 166 294 294 294 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 166 166 166 294 294 294 F F F 20 400-419 165 165 165 293 293 293 F F F 21 420-439 168 168 168 298 298 298 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 165 165 165 293 293 293 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 168 168 168 296 296 296 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 166 166 166 294 294 294 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 168, Max flush: 298 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 168 168 168 295 295 295 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 166 166 166 294 294 294 F F F 4 080-099 169 169 169 299 299 299 F F F 5 100-119 169 169 169 299 299 299 F F F 6 120-139 166 166 166 294 294 294 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 168 168 168 297 297 297 F F F 9 180-199 169 169 169 299 299 299 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 168 168 168 297 297 297 F F F 13 260-279 168 168 168 297 297 297 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 166 166 166 294 294 294 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 166 166 166 294 294 294 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 166 166 166 294 294 294 F F F 20 400-419 165 165 165 293 293 293 F F F 21 420-439 168 168 168 298 298 298 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 165 165 165 293 293 293 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 168 168 168 296 296 296 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 166 166 166 294 294 294 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 169, Max flush: 299 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.53687163Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.580581385Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.58121356Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.581248881Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.581263271Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.581277831Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862} {"msg":"Crucible f5106acc-2b8f-44f3-8ba4-e20359d64d44 has session id: 6a24d9cb-2ee9-4e44-be9b-58e673d6731d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.581723492Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Upstairs opts: Upstairs UUID: f5106acc-2b8f-44f3-8ba4-e20359d64d44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.581757483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Crucible stats registered with UUID: f5106acc-2b8f-44f3-8ba4-e20359d64d44","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.581773133Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611239108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611271208Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862} {"msg":"f5106acc-2b8f-44f3-8ba4-e20359d64d44 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611297609Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611314549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.61134382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.61136226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611379781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611396431Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611413812Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611430502Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611447933Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611465053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42923)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611486183Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48213)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611520084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47355)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.611556685Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.612389935Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.612419066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.612434356Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.612500588Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.612550229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.612567039Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613072101Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613103192Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613121182Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613592324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613620225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613635055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613708537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613728667Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613742517Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613754848Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613768168Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[0]R flush_numbers[0..12]: [275, 295, 288, 294, 299, 299, 294, 288, 297, 299, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613782078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[0]R generation[0..12]: [157, 168, 162, 166, 169, 169, 166, 162, 168, 169, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613799629Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613815179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[1]R flush_numbers[0..12]: [275, 295, 288, 294, 299, 299, 294, 288, 297, 299, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613829399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[1]R generation[0..12]: [157, 168, 162, 166, 169, 169, 166, 162, 168, 169, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.61384322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.61385744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[2]R flush_numbers[0..12]: [275, 295, 288, 294, 299, 299, 294, 288, 297, 299, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613871511Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[2]R generation[0..12]: [157, 168, 162, 166, 169, 169, 166, 162, 168, 169, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613885351Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613899131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613912461Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613925922Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613946912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613959963Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"":"downstairs","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613973553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.613986153Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"0","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.614001904Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"1","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.614015634Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"client":"2","":"downstairs","session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.614029494Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.614042015Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"f5106acc-2b8f-44f3-8ba4-e20359d64d44 is now active with session: 6a24d9cb-2ee9-4e44-be9b-58e673d6731d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.614054855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.614066985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862,"session_id":"6a24d9cb-2ee9-4e44-be9b-58e673d6731d"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:53.614079316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 16384 data: 10 10 9 12 02/30 Read block 263 len 4096 03/30 Flush 04/30 Write block 476 len 20480 data: 7 7 10 13 15 05/30 Write block 531 len 24576 data: 10 12 11 10 12 13 06/30 Read block 292 len 24576 07/30 Write block 397 len 32768 data: 13 12 9 12 14 18 20 16 08/30 Read block 569 len 4096 09/30 Flush 10/30 Write block 75 len 20480 data: 8 8 13 12 9 11/30 Read block 348 len 4096 12/30 Flush 13/30 Write block 109 len 8192 data: 9 9 14/30 Read block 253 len 24576 15/30 Read block 322 len 24576 16/30 Write block 403 len 12288 data: 21 17 13 17/30 Write block 389 len 40960 data: 13 13 12 12 12 13 13 14 14 13 18/30 Write block 95 len 24576 data: 15 15 13 15 18 20 19/30 Read block 262 len 28672 20/30 Flush 21/30 Write block 555 len 40960 data: 12 12 16 12 9 8 6 8 9 7 22/30 Read block 35 len 24576 23/30 Read block 129 len 32768 24/30 Read block 201 len 28672 25/30 Write block 351 len 20480 data: 15 14 13 15 11 26/30 Read block 176 len 12288 27/30 Read block 109 len 32768 28/30 Read block 411 len 24576 29/30 Read block 553 len 40960 30/30 Write block 393 len 40960 data: 13 14 14 15 15 14 10 13 15 19 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:53.634285768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 168 168 168 295 295 295 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 170 170 170 302 302 302 F F F 4 080-099 170 170 170 303 303 303 F F F 5 100-119 170 170 170 303 303 303 F F F 6 120-139 166 166 166 294 294 294 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 168 168 168 297 297 297 F F F 9 180-199 169 169 169 299 299 299 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 168 168 168 297 297 297 F F F 13 260-279 168 168 168 297 297 297 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 166 166 166 294 294 294 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 170 170 170 300 300 300 T T T 18 360-379 166 166 166 294 294 294 F F F 19 380-399 170 170 170 303 303 303 T T T 20 400-419 170 170 170 303 303 303 T T T 21 420-439 168 168 168 298 298 298 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 170 170 170 301 301 301 F F F 25 500-519 168 168 168 296 296 296 F F F 26 520-539 170 170 170 301 301 301 F F F 27 540-559 161 161 161 285 285 285 T T T 28 560-579 166 166 166 294 294 294 T T T 29 580-599 162 162 162 289 289 289 F F F Max gen: 170, Max flush: 303 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 168 168 168 295 295 295 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 170 170 170 302 302 302 F F F 4 080-099 170 170 170 303 303 303 F F F 5 100-119 170 170 170 303 303 303 F F F 6 120-139 166 166 166 294 294 294 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 168 168 168 297 297 297 F F F 9 180-199 169 169 169 299 299 299 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 168 168 168 297 297 297 F F F 13 260-279 168 168 168 297 297 297 F F F 14 280-299 164 164 164 291 291 291 F F F 15 300-319 166 166 166 294 294 294 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 171 171 171 304 304 304 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 171 171 171 304 304 304 F F F 21 420-439 168 168 168 298 298 298 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 170 170 170 301 301 301 F F F 25 500-519 168 168 168 296 296 296 F F F 26 520-539 170 170 170 301 301 301 F F F 27 540-559 171 171 171 304 304 304 F F F 28 560-579 171 171 171 304 304 304 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 171, Max flush: 304 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.889324601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.93278026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.933459566Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.933497287Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.933513048Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.933528308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870} {"msg":"Crucible 1626f2c1-5817-455f-a1e3-2573d04e29e3 has session id: 4b3c915c-c0fc-4188-9554-b741433602a5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.933955238Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Upstairs opts: Upstairs UUID: 1626f2c1-5817-455f-a1e3-2573d04e29e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.933989379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Crucible stats registered with UUID: 1626f2c1-5817-455f-a1e3-2573d04e29e3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.934004589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.96372984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.963760551Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870} {"msg":"1626f2c1-5817-455f-a1e3-2573d04e29e3 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.963831532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.963854763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.963883354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.963911264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.963944615Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.963976386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.964010377Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.964044438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.964073858Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.964102469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61062)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.96413757Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46256)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.964170781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39303)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.964218022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.964956379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.96498594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.965036881Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.965053102Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.965074322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.965101503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.965660076Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.965690447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.965706767Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.96625568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966283871Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966298581Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966311102Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966324682Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966338642Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966351203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966364663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[0]R flush_numbers[0..12]: [275, 295, 288, 302, 303, 303, 294, 288, 297, 299, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966378713Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[0]R generation[0..12]: [157, 168, 162, 170, 170, 170, 166, 162, 168, 169, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966392824Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966407214Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[1]R flush_numbers[0..12]: [275, 295, 288, 302, 303, 303, 294, 288, 297, 299, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966424654Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[1]R generation[0..12]: [157, 168, 162, 170, 170, 170, 166, 162, 168, 169, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966439245Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966453605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[2]R flush_numbers[0..12]: [275, 295, 288, 302, 303, 303, 294, 288, 297, 299, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966467505Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[2]R generation[0..12]: [157, 168, 162, 170, 170, 170, 166, 162, 168, 169, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966481526Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966495696Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:13:57.966509176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966530427Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966543657Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966556927Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"":"downstairs","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966570658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966582868Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"0","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966596468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"1","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966609999Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"client":"2","":"downstairs","session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.966623709Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.96663836Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"1626f2c1-5817-455f-a1e3-2573d04e29e3 is now active with session: 4b3c915c-c0fc-4188-9554-b741433602a5","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.9666518Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.96666408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870,"session_id":"4b3c915c-c0fc-4188-9554-b741433602a5"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:13:57.96667729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 136 len 4096 data: 9 02/30 Read block 242 len 8192 03/30 Write block 181 len 28672 data: 11 11 11 12 12 13 13 04/30 Read block 446 len 20480 05/30 Write block 506 len 12288 data: 16 17 14 06/30 Write block 491 len 8192 data: 12 11 07/30 Write block 298 len 20480 data: 16 11 9 8 6 08/30 Read block 207 len 12288 09/30 Flush 10/30 Write block 501 len 32768 data: 16 17 19 17 14 17 18 15 11/30 Read block 281 len 36864 12/30 Flush 13/30 Flush 14/30 Write block 364 len 20480 data: 12 13 14 13 15 15/30 Read block 511 len 32768 16/30 Read block 329 len 24576 17/30 Read block 481 len 8192 18/30 Flush 19/30 Write block 85 len 24576 data: 8 6 6 10 13 15 20/30 Read block 407 len 16384 21/30 Flush 22/30 Write block 284 len 4096 data: 14 23/30 Flush 24/30 Write block 508 len 24576 data: 16 13 14 13 12 12 25/30 Flush 26/30 Write block 362 len 40960 data: 14 14 13 14 15 14 16 16 11 10 27/30 Read block 354 len 32768 28/30 Read block 172 len 4096 29/30 Read block 294 len 16384 30/30 Read block 64 len 4096 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:13:57.985178763Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 168 168 168 295 295 295 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 170 170 170 302 302 302 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 170 170 170 303 303 303 F F F 6 120-139 172 172 172 305 305 305 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 168 168 168 297 297 297 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 168 168 168 297 297 297 F F F 13 260-279 168 168 168 297 297 297 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 172 172 172 305 305 305 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 171 171 171 304 304 304 F F F 18 360-379 172 172 172 308 308 308 T T T 19 380-399 171 171 171 304 304 304 F F F 20 400-419 171 171 171 304 304 304 F F F 21 420-439 168 168 168 298 298 298 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 172 172 172 305 305 305 F F F 25 500-519 172 172 172 311 311 311 F F F 26 520-539 170 170 170 301 301 301 F F F 27 540-559 171 171 171 304 304 304 F F F 28 560-579 171 171 171 304 304 304 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 172, Max flush: 311 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 168 168 168 295 295 295 F F F 2 040-059 162 162 162 288 288 288 F F F 3 060-079 170 170 170 302 302 302 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 170 170 170 303 303 303 F F F 6 120-139 172 172 172 305 305 305 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 168 168 168 297 297 297 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 168 168 168 297 297 297 F F F 13 260-279 168 168 168 297 297 297 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 172 172 172 305 305 305 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 171 171 171 304 304 304 F F F 18 360-379 173 173 173 312 312 312 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 171 171 171 304 304 304 F F F 21 420-439 168 168 168 298 298 298 F F F 22 440-459 164 164 164 291 291 291 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 172 172 172 305 305 305 F F F 25 500-519 172 172 172 311 311 311 F F F 26 520-539 170 170 170 301 301 301 F F F 27 540-559 171 171 171 304 304 304 F F F 28 560-579 171 171 171 304 304 304 F F F 29 580-599 162 162 162 289 289 289 F F F Max gen: 173, Max flush: 312 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.224559221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.267473787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.268074271Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.268109902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.268129292Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.268144953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878} {"msg":"Crucible 436290dd-1f60-4a87-b602-7fcbe4602dfb has session id: 8d9bb2b6-81f7-4024-ad85-13739c4e67ec","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.268527652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Upstairs opts: Upstairs UUID: 436290dd-1f60-4a87-b602-7fcbe4602dfb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.268571733Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Crucible stats registered with UUID: 436290dd-1f60-4a87-b602-7fcbe4602dfb","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.268597304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298325474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298360855Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878} {"msg":"436290dd-1f60-4a87-b602-7fcbe4602dfb active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298394946Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298413407Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298442637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298460877Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298478248Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298495018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298512009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298528829Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.29855058Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.29856897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:52016)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298744804Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:42422)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298786625Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:57479)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.298819416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.299465612Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.299495272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.299510573Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.299533563Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.299561204Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.299589884Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300113307Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300144058Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300160468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.30065747Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300690001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300705571Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300790723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300814014Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300828304Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300841625Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300855535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[0]R flush_numbers[0..12]: [275, 295, 288, 302, 309, 303, 305, 288, 297, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300869935Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[0]R generation[0..12]: [157, 168, 162, 170, 172, 170, 172, 162, 168, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300884335Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300898656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[1]R flush_numbers[0..12]: [275, 295, 288, 302, 309, 303, 305, 288, 297, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300913396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[1]R generation[0..12]: [157, 168, 162, 170, 172, 170, 172, 162, 168, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300927507Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300941537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[2]R flush_numbers[0..12]: [275, 295, 288, 302, 309, 303, 305, 288, 297, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300956087Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[2]R generation[0..12]: [157, 168, 162, 170, 172, 170, 172, 162, 168, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300973348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.300988018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301001408Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Generation requested: 174 >= found:174"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301024049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301037399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.30105055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"":"downstairs","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.30106477Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.30107745Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"0","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.30109092Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"1","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301104441Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"client":"2","":"downstairs","session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301117961Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301130561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"436290dd-1f60-4a87-b602-7fcbe4602dfb is now active with session: 8d9bb2b6-81f7-4024-ad85-13739c4e67ec","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-12-09T15:14:02.301143342Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301156822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878,"session_id":"8d9bb2b6-81f7-4024-ad85-13739c4e67ec"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:02.301169612Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 43 len 16384 data: 16 16 16 17 02/30 Write block 571 len 24576 data: 15 16 14 17 18 16 03/30 Write block 272 len 16384 data: 15 15 12 12 04/30 Read block 347 len 16384 05/30 Read block 24 len 4096 06/30 Read block 109 len 8192 07/30 Flush 08/30 Write block 294 len 12288 data: 18 19 19 09/30 Write block 21 len 4096 data: 12 10/30 Read block 533 len 16384 11/30 Write block 546 len 8192 data: 10 11 12/30 Write block 251 len 36864 data: 8 10 11 12 12 11 11 12 12 13/30 Read block 150 len 24576 14/30 Read block 347 len 36864 15/30 Write block 353 len 24576 data: 14 16 12 11 12 10 16/30 Write block 578 len 32768 data: 16 15 15 13 12 13 12 10 17/30 Write block 288 len 16384 data: 15 14 16 19 18/30 Read block 529 len 8192 19/30 Read block 151 len 28672 20/30 Write block 166 len 40960 data: 14 8 8 7 6 6 8 13 13 12 21/30 Read block 248 len 8192 22/30 Read block 550 len 36864 23/30 Write block 428 len 28672 data: 18 17 16 13 12 11 13 24/30 Write block 443 len 24576 data: 10 11 10 11 9 9 25/30 Write block 76 len 32768 data: 9 14 13 10 7 11 10 9 26/30 Write block 251 len 36864 data: 9 11 12 13 13 12 12 13 13 27/30 Read block 168 len 4096 28/30 Write block 113 len 12288 data: 9 8 8 29/30 Read block 414 len 12288 30/30 Write block 340 len 28672 data: 13 11 11 10 13 14 18 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 3 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:02.319865279Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 168 168 168 295 295 295 T T T 2 040-059 174 174 174 313 313 313 F F F 3 060-079 170 170 170 302 302 302 T T T 4 080-099 172 172 172 309 309 309 T T T 5 100-119 170 170 170 303 303 303 T T T 6 120-139 172 172 172 305 305 305 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 168 168 168 297 297 297 T T T 9 180-199 172 172 172 305 305 305 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 168 168 168 297 297 297 T T T 13 260-279 174 174 174 313 313 313 F F F 14 280-299 172 172 172 310 310 310 T T T 15 300-319 172 172 172 305 305 305 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 171 171 171 304 304 304 T T T 18 360-379 173 173 173 312 312 312 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 171 171 171 304 304 304 F F F 21 420-439 168 168 168 298 298 298 T T T 22 440-459 164 164 164 291 291 291 T T T 23 460-479 170 170 170 301 301 301 F F F 24 480-499 172 172 172 305 305 305 F F F 25 500-519 172 172 172 311 311 311 F F F 26 520-539 170 170 170 301 301 301 F F F 27 540-559 171 171 171 304 304 304 T T T 28 560-579 174 174 174 313 313 313 T T T 29 580-599 162 162 162 289 289 289 T T T Max gen: 174, Max flush: 313 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 175 175 175 314 314 314 F F F 2 040-059 174 174 174 313 313 313 F F F 3 060-079 175 175 175 314 314 314 F F F 4 080-099 175 175 175 314 314 314 F F F 5 100-119 175 175 175 314 314 314 F F F 6 120-139 172 172 172 305 305 305 F F F 7 140-159 162 162 162 288 288 288 F F F 8 160-179 175 175 175 314 314 314 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 175 175 175 314 314 314 F F F 13 260-279 174 174 174 313 313 313 F F F 14 280-299 175 175 175 314 314 314 F F F 15 300-319 172 172 172 305 305 305 F F F 16 320-339 168 168 168 297 297 297 F F F 17 340-359 175 175 175 314 314 314 F F F 18 360-379 173 173 173 312 312 312 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 171 171 171 304 304 304 F F F 21 420-439 175 175 175 314 314 314 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 172 172 172 305 305 305 F F F 25 500-519 172 172 172 311 311 311 F F F 26 520-539 170 170 170 301 301 301 F F F 27 540-559 175 175 175 314 314 314 F F F 28 560-579 175 175 175 314 314 314 F F F 29 580-599 175 175 175 314 314 314 F F F Max gen: 175, Max flush: 314 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.629783144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.673613482Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.674219546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.674248097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.674264537Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.674279297Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886} {"msg":"Crucible f5e24032-53d6-4a33-9c20-dd88de115662 has session id: 589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.674704558Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Upstairs opts: Upstairs UUID: f5e24032-53d6-4a33-9c20-dd88de115662, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.674746789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Crucible stats registered with UUID: f5e24032-53d6-4a33-9c20-dd88de115662","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.674764659Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704448149Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.70448097Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886} {"msg":"f5e24032-53d6-4a33-9c20-dd88de115662 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704551591Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704574122Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704605282Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704624813Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704643273Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704660624Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704678104Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704699525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704731505Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704767866Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62151)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704802937Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37565)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704841708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42649)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.704878859Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.705553585Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.705596136Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.705666378Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.705703169Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.705733829Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.70576499Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.70618064Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706316133Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706420896Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706723163Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706753004Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706768214Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706854276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706872967Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706886727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706941248Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706959909Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[0]R flush_numbers[0..12]: [275, 314, 313, 314, 314, 314, 305, 288, 314, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706974289Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[0]R generation[0..12]: [157, 175, 174, 175, 175, 175, 172, 162, 175, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.706988739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.70700319Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[1]R flush_numbers[0..12]: [275, 314, 313, 314, 314, 314, 305, 288, 314, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.70701742Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[1]R generation[0..12]: [157, 175, 174, 175, 175, 175, 172, 162, 175, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.70703118Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707045331Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[2]R flush_numbers[0..12]: [275, 314, 313, 314, 314, 314, 305, 288, 314, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707059151Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[2]R generation[0..12]: [157, 175, 174, 175, 175, 175, 172, 162, 175, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707076312Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707090832Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707104322Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707118073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707131333Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707144343Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"":"downstairs","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:14:06.707157754Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707175264Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"0","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707188574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"1","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707202004Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"client":"2","":"downstairs","session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707215395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707227955Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"f5e24032-53d6-4a33-9c20-dd88de115662 is now active with session: 589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707258866Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707271546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886,"session_id":"589cd577-c2a7-4cfe-bbd2-0c1fc917dc4f"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:06.707284217Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 359 len 24576 data: 12 13 15 15 15 14 02/30 Read block 184 len 36864 03/30 Write block 371 len 12288 data: 11 11 9 04/30 Flush 05/30 Write block 580 len 12288 data: 16 14 13 06/30 Write block 325 len 12288 data: 16 17 15 07/30 Read block 221 len 40960 08/30 Write block 284 len 36864 data: 15 14 16 17 16 15 17 20 23 09/30 Read block 381 len 12288 10/30 Read block 23 len 8192 11/30 Write block 515 len 4096 data: 16 12/30 Write block 480 len 16384 data: 16 19 17 15 13/30 Flush 14/30 Flush 15/30 Write block 129 len 40960 data: 10 10 8 9 8 7 8 10 6 9 16/30 Write block 411 len 36864 data: 18 17 18 16 16 14 13 12 12 17/30 Write block 106 len 16384 data: 10 10 8 10 18/30 Read block 21 len 8192 19/30 Read block 61 len 36864 20/30 Write block 517 len 36864 data: 13 11 8 9 8 9 9 8 8 21/30 Write block 570 len 8192 data: 12 16 22/30 Flush 23/30 Write block 307 len 4096 data: 12 24/30 Flush 25/30 Write block 35 len 36864 data: 15 19 19 19 16 14 15 15 17 26/30 Write block 103 len 28672 data: 13 13 12 11 11 9 11 27/30 Read block 450 len 8192 28/30 Read block 360 len 8192 29/30 Read block 487 len 12288 30/30 Write block 157 len 16384 data: 21 23 25 21 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:06.725790689Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 175 175 175 314 314 314 T T T 2 040-059 174 174 174 313 313 313 T T T 3 060-079 175 175 175 314 314 314 F F F 4 080-099 175 175 175 314 314 314 F F F 5 100-119 176 176 176 318 318 318 T T T 6 120-139 176 176 176 318 318 318 F F F 7 140-159 162 162 162 288 288 288 T T T 8 160-179 175 175 175 314 314 314 T T T 9 180-199 172 172 172 305 305 305 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 175 175 175 314 314 314 F F F 13 260-279 174 174 174 313 313 313 F F F 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 176 176 176 316 316 316 F F F 17 340-359 176 176 176 315 315 315 F F F 18 360-379 176 176 176 315 315 315 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 175 175 175 314 314 314 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 176 176 176 318 318 318 F F F 26 520-539 176 176 176 318 318 318 F F F 27 540-559 175 175 175 314 314 314 F F F 28 560-579 176 176 176 318 318 318 F F F 29 580-599 176 176 176 316 316 316 F F F Max gen: 176, Max flush: 319 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 F F F 1 020-039 177 177 177 320 320 320 F F F 2 040-059 177 177 177 320 320 320 F F F 3 060-079 175 175 175 314 314 314 F F F 4 080-099 175 175 175 314 314 314 F F F 5 100-119 177 177 177 320 320 320 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 177 177 177 320 320 320 F F F 8 160-179 177 177 177 320 320 320 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 169 169 169 299 299 299 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 175 175 175 314 314 314 F F F 13 260-279 174 174 174 313 313 313 F F F 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 176 176 176 316 316 316 F F F 17 340-359 176 176 176 315 315 315 F F F 18 360-379 176 176 176 315 315 315 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 175 175 175 314 314 314 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 176 176 176 318 318 318 F F F 26 520-539 176 176 176 318 318 318 F F F 27 540-559 175 175 175 314 314 314 F F F 28 560-579 176 176 176 318 318 318 F F F 29 580-599 176 176 176 316 316 316 F F F Max gen: 177, Max flush: 320 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:10.986540278Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.029253059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.029862544Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.029897635Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.029912175Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.029926795Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894} {"msg":"Crucible 3ff36cec-7458-4bb3-9860-f4f8b6ecea77 has session id: 36f4d055-ca5d-4737-b381-c17208ad3174","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.030379336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Upstairs opts: Upstairs UUID: 3ff36cec-7458-4bb3-9860-f4f8b6ecea77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.030425878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Crucible stats registered with UUID: 3ff36cec-7458-4bb3-9860-f4f8b6ecea77","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.030451278Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.059982534Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060025605Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894} {"msg":"3ff36cec-7458-4bb3-9860-f4f8b6ecea77 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060067096Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060085067Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060114857Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060131357Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060145478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060158878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060172469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060185909Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060199929Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060214019Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_connection connected from Ok(127.0.0.1:61676)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060289781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_connection connected from Ok(127.0.0.1:38719)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060491666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_connection connected from Ok(127.0.0.1:50887)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.060532967Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061123851Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061152162Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061167332Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061181363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061197643Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061223514Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061874679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.06190463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.061923471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062483984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062512065Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062526555Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062539205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062556055Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062570016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062582486Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062595776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[0]R flush_numbers[0..12]: [275, 320, 320, 314, 314, 320, 318, 320, 320, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062609707Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[0]R generation[0..12]: [157, 177, 177, 175, 175, 177, 176, 177, 177, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062623717Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062637848Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[1]R flush_numbers[0..12]: [275, 320, 320, 314, 314, 320, 318, 320, 320, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062651688Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[1]R generation[0..12]: [157, 177, 177, 175, 175, 177, 176, 177, 177, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062665568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062679329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[2]R flush_numbers[0..12]: [275, 320, 320, 314, 314, 320, 318, 320, 320, 305, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062693089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[2]R generation[0..12]: [157, 177, 177, 175, 175, 177, 176, 177, 177, 172, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062706789Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.06272059Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.06273388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-12-09T15:14:11.06275249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062830182Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062846562Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"":"downstairs","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062859793Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062871963Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"0","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062885354Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"1","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062898574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"client":"2","":"downstairs","session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062913064Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062925794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"3ff36cec-7458-4bb3-9860-f4f8b6ecea77 is now active with session: 36f4d055-ca5d-4737-b381-c17208ad3174","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062938755Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062950875Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894,"session_id":"36f4d055-ca5d-4737-b381-c17208ad3174"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:11.062963405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 12288 data: 17 13 10 02/30 Write block 258 len 8192 data: 14 14 03/30 Read block 239 len 8192 04/30 Write block 217 len 8192 data: 8 8 05/30 Read block 64 len 20480 06/30 Read block 93 len 32768 07/30 Read block 159 len 12288 08/30 Write block 342 len 28672 data: 12 11 14 15 19 20 18 09/30 Read block 101 len 20480 10/30 Write block 553 len 24576 data: 14 15 13 13 18 14 11/30 Flush 12/30 Write block 78 len 4096 data: 14 13/30 Write block 101 len 32768 data: 18 15 14 14 13 12 12 10 14/30 Read block 333 len 4096 15/30 Write block 431 len 24576 data: 14 13 12 14 12 11 16/30 Flush 17/30 Write block 137 len 12288 data: 7 10 11 18/30 Read block 322 len 20480 19/30 Write block 1 len 24576 data: 5 6 8 10 10 10 20/30 Write block 137 len 40960 data: 8 11 12 15 18 15 13 13 10 11 21/30 Write block 98 len 28672 data: 16 19 21 19 16 15 15 22/30 Write block 367 len 40960 data: 15 17 17 12 12 12 10 8 8 10 23/30 Read block 564 len 36864 24/30 Read block 244 len 36864 25/30 Read block 582 len 12288 26/30 Write block 537 len 20480 data: 16 16 16 19 15 27/30 Write block 36 len 36864 data: 20 20 20 17 15 16 16 18 17 28/30 Read block 402 len 8192 29/30 Read block 573 len 36864 30/30 Read block 400 len 20480 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:11.082515753Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 275 275 275 T T T 1 020-039 177 177 177 320 320 320 T T T 2 040-059 177 177 177 320 320 320 T T T 3 060-079 178 178 178 322 322 322 F F F 4 080-099 175 175 175 314 314 314 T T T 5 100-119 178 178 178 322 322 322 T T T 6 120-139 176 176 176 318 318 318 T T T 7 140-159 177 177 177 320 320 320 T T T 8 160-179 177 177 177 320 320 320 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 178 178 178 321 321 321 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 174 174 174 313 313 313 F F F 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 176 176 176 316 316 316 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 176 176 176 315 315 315 T T T 19 380-399 171 171 171 304 304 304 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 178 178 178 322 322 322 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 176 176 176 318 318 318 F F F 26 520-539 176 176 176 318 318 318 T T T 27 540-559 178 178 178 321 321 321 T T T 28 560-579 176 176 176 318 318 318 F F F 29 580-599 176 176 176 316 316 316 F F F Max gen: 178, Max flush: 322 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 323 323 323 F F F 1 020-039 179 179 179 323 323 323 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 178 178 178 322 322 322 F F F 4 080-099 179 179 179 323 323 323 F F F 5 100-119 179 179 179 323 323 323 F F F 6 120-139 179 179 179 323 323 323 F F F 7 140-159 179 179 179 323 323 323 F F F 8 160-179 177 177 177 320 320 320 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 178 178 178 321 321 321 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 174 174 174 313 313 313 F F F 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 176 176 176 316 316 316 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 179 179 179 323 323 323 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 178 178 178 322 322 322 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 170 170 170 301 301 301 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 176 176 176 318 318 318 F F F 26 520-539 179 179 179 323 323 323 F F F 27 540-559 179 179 179 323 323 323 F F F 28 560-579 176 176 176 318 318 318 F F F 29 580-599 176 176 176 316 316 316 F F F Max gen: 179, Max flush: 323 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.376548167Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.419144756Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.419711419Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.41974708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.41976174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.419776301Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902} {"msg":"Crucible 2925de35-2f89-45cd-92c4-e4a69565bc1b has session id: 09b3f925-cd29-444e-9901-046f68459426","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.420209801Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Upstairs opts: Upstairs UUID: 2925de35-2f89-45cd-92c4-e4a69565bc1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.420250632Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Crucible stats registered with UUID: 2925de35-2f89-45cd-92c4-e4a69565bc1b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.420270842Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450152487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450195968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902} {"msg":"2925de35-2f89-45cd-92c4-e4a69565bc1b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450260339Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.4502895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450316231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450332241Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450346641Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450361062Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450374952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450398963Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450428874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450459934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_connection connected from Ok(127.0.0.1:34993)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450500195Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_connection connected from Ok(127.0.0.1:52610)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.450531136Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_connection connected from Ok(127.0.0.1:62992)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.45069298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.451294504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.451323255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.451339345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.451354016Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.451396787Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.451427277Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.451910399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452045902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452209006Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452424891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452461042Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452478383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452491213Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452505663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452523744Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452676987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452697178Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[0]R flush_numbers[0..12]: [323, 323, 323, 322, 323, 323, 323, 323, 320, 305, 321, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452711648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[0]R generation[0..12]: [179, 179, 179, 178, 179, 179, 179, 179, 177, 172, 178, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452725808Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452739919Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[1]R flush_numbers[0..12]: [323, 323, 323, 322, 323, 323, 323, 323, 320, 305, 321, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452753729Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[1]R generation[0..12]: [179, 179, 179, 178, 179, 179, 179, 179, 177, 172, 178, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452767369Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.45278125Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[2]R flush_numbers[0..12]: [323, 323, 323, 322, 323, 323, 323, 323, 320, 305, 321, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.45279523Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[2]R generation[0..12]: [179, 179, 179, 178, 179, 179, 179, 179, 177, 172, 178, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.45280899Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452822821Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452835891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452849221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452862272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:14:15.452877892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"":"downstairs","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452899113Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452911393Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"0","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452924733Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"1","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452937934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"client":"2","":"downstairs","session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452951234Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452963794Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"2925de35-2f89-45cd-92c4-e4a69565bc1b is now active with session: 09b3f925-cd29-444e-9901-046f68459426","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452976644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.452988945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902,"session_id":"09b3f925-cd29-444e-9901-046f68459426"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:15.453001345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 165 len 36864 data: 16 15 9 9 8 7 7 9 14 02/30 Read block 101 len 12288 03/30 Read block 198 len 4096 04/30 Read block 374 len 40960 05/30 Write block 116 len 24576 data: 11 12 9 14 15 14 06/30 Read block 216 len 4096 07/30 Flush 08/30 Write block 276 len 8192 data: 8 7 09/30 Read block 25 len 4096 10/30 Read block 84 len 36864 11/30 Write block 508 len 16384 data: 17 14 15 14 12/30 Write block 476 len 4096 data: 8 13/30 Read block 525 len 4096 14/30 Read block 84 len 28672 15/30 Write block 93 len 12288 data: 17 13 16 16/30 Write block 269 len 40960 data: 20 18 13 16 16 13 13 9 8 7 17/30 Read block 555 len 16384 18/30 Read block 405 len 16384 19/30 Write block 503 len 32768 data: 20 18 15 18 19 18 15 16 20/30 Read block 27 len 16384 21/30 Read block 31 len 24576 22/30 Read block 250 len 24576 23/30 Write block 61 len 8192 data: 16 15 24/30 Write block 77 len 12288 data: 15 15 11 25/30 Write block 176 len 16384 data: 13 10 13 13 26/30 Read block 3 len 36864 27/30 Read block 153 len 24576 28/30 Read block 175 len 40960 29/30 Write block 116 len 8192 data: 12 13 30/30 Write block 325 len 20480 data: 17 18 16 14 13 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:15.471903057Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 323 323 323 F F F 1 020-039 179 179 179 323 323 323 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 178 178 178 322 322 322 T T T 4 080-099 179 179 179 323 323 323 T T T 5 100-119 180 180 180 324 324 324 T T T 6 120-139 180 180 180 324 324 324 F F F 7 140-159 179 179 179 323 323 323 F F F 8 160-179 180 180 180 324 324 324 T T T 9 180-199 172 172 172 305 305 305 F F F 10 200-219 178 178 178 321 321 321 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 174 174 174 313 313 313 T T T 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 176 176 176 316 316 316 T T T 17 340-359 178 178 178 321 321 321 F F F 18 360-379 179 179 179 323 323 323 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 178 178 178 322 322 322 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 170 170 170 301 301 301 T T T 24 480-499 176 176 176 316 316 316 F F F 25 500-519 176 176 176 318 318 318 T T T 26 520-539 179 179 179 323 323 323 F F F 27 540-559 179 179 179 323 323 323 F F F 28 560-579 176 176 176 318 318 318 F F F 29 580-599 176 176 176 316 316 316 F F F Max gen: 180, Max flush: 324 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 323 323 323 F F F 1 020-039 179 179 179 323 323 323 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 181 181 181 325 325 325 F F F 4 080-099 181 181 181 325 325 325 F F F 5 100-119 181 181 181 325 325 325 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 179 179 179 323 323 323 F F F 8 160-179 181 181 181 325 325 325 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 178 178 178 321 321 321 F F F 11 220-239 166 166 166 294 294 294 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 181 181 181 325 325 325 F F F 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 181 181 181 325 325 325 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 179 179 179 323 323 323 F F F 19 380-399 171 171 171 304 304 304 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 178 178 178 322 322 322 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 181 181 181 325 325 325 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 181 181 181 325 325 325 F F F 26 520-539 179 179 179 323 323 323 F F F 27 540-559 179 179 179 323 323 323 F F F 28 560-579 176 176 176 318 318 318 F F F 29 580-599 176 176 176 316 316 316 F F F Max gen: 181, Max flush: 325 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.754832626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:19.796195474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.796584244Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.827014881Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.827601505Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.827638076Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.827652626Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.827668797Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910} {"msg":"Crucible b5f08410-b66f-478a-bfd7-9da98b8a4ca0 has session id: 5b8353cc-d97e-46f0-ba73-2d9dd0a383a7","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.828073926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Upstairs opts: Upstairs UUID: b5f08410-b66f-478a-bfd7-9da98b8a4ca0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.828122038Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Crucible stats registered with UUID: b5f08410-b66f-478a-bfd7-9da98b8a4ca0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.828141288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.858885933Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.858917984Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910} {"msg":"b5f08410-b66f-478a-bfd7-9da98b8a4ca0 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.858989915Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859013206Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859051737Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859086398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859123348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.85916012Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.8591977Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859233651Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859269142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859391325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43051)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859433726Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62450)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859473867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:52258)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.859504438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860053881Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860095542Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860125892Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860156923Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860186784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860216835Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860664245Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860695056Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.860836409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861246229Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.86127604Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.86129105Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861359022Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861392663Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861409883Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861422943Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861436754Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[0]R flush_numbers[0..12]: [323, 323, 323, 325, 325, 325, 324, 323, 325, 305, 321, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861450644Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[0]R generation[0..12]: [179, 179, 179, 181, 181, 181, 180, 179, 181, 172, 178, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861464784Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861480135Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[1]R flush_numbers[0..12]: [323, 323, 323, 325, 325, 325, 324, 323, 325, 305, 321, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861494395Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[1]R generation[0..12]: [179, 179, 179, 181, 181, 181, 180, 179, 181, 172, 178, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861508545Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861522796Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[2]R flush_numbers[0..12]: [323, 323, 323, 325, 325, 325, 324, 323, 325, 305, 321, 294]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861536666Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[2]R generation[0..12]: [179, 179, 179, 181, 181, 181, 180, 179, 181, 172, 178, 166]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861550446Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861564437Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861577627Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861593207Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861614828Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861628359Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"":"downstairs","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861641779Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861654009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"0","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861667409Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"1","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.86168085Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"client":"2","":"downstairs","session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.86169454Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.86170707Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"b5f08410-b66f-478a-bfd7-9da98b8a4ca0 is now active with session: 5b8353cc-d97e-46f0-ba73-2d9dd0a383a7","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861720161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861732411Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910,"session_id":"5b8353cc-d97e-46f0-ba73-2d9dd0a383a7Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:19.861749761Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 508 len 40960 data: 19 16 17 15 13 13 11 17 14 14 02/30 Read block 7 len 20480 03/30 Write block 104 len 8192 data: 16 14 04/30 Write block 384 len 16384 data: 15 12 12 15 05/30 Read block 397 len 12288 06/30 Write block 362 len 12288 data: 16 16 15 07/30 Write block 466 len 36864 data: 9 9 9 10 10 10 8 9 5 08/30 Write block 399 len 24576 data: 11 14 16 20 22 18 09/30 Write block 294 len 16384 data: 19 20 20 21 10/30 Read block 321 len 32768 11/30 Read block 185 len 24576 12/30 Write block 360 len 24576 data: 14 16 17 17 16 15 13/30 Write block 212 len 36864 data: 9 10 7 7 9 9 9 11 13 14/30 Write block 380 len 32768 data: 13 15 16 15 16 13 13 16 15/30 Write block 35 len 16384 data: 16 21 21 21 16/30 Read block 546 len 40960 17/30 Write block 382 len 32768 data: 17 16 17 14 14 17 12 14 18/30 Write block 313 len 12288 data: 15 17 18 19/30 Write block 538 len 36864 data: 17 17 20 16 16 16 16 13 11 20/30 Read block 341 len 4096 21/30 Read block 563 len 28672 22/30 Flush 23/30 Write block 268 len 32768 data: 17 21 19 14 17 17 14 14 24/30 Write block 268 len 40960 data: 18 22 20 15 18 18 15 15 10 9 25/30 Write block 503 len 40960 data: 21 19 16 19 20 20 17 18 16 14 26/30 Write block 301 len 20480 data: 9 7 9 7 8 27/30 Write block 386 len 28672 data: 15 18 13 15 14 13 13 28/30 Write block 595 len 4096 data: 9 29/30 Read block 440 len 32768 30/30 Read block 129 len 28672 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 7 8 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:19.880816157Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 323 323 323 F F F 1 020-039 182 182 182 326 326 326 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 181 181 181 325 325 325 F F F 4 080-099 181 181 181 325 325 325 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 179 179 179 323 323 323 F F F 8 160-179 181 181 181 325 325 325 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 182 182 182 326 326 326 F F F 11 220-239 182 182 182 326 326 326 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 181 181 181 325 325 325 T T T 14 280-299 182 182 182 326 326 326 F F F 15 300-319 182 182 182 326 326 326 T T T 16 320-339 181 181 181 325 325 325 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 182 182 182 326 326 326 T T T 20 400-419 182 182 182 326 326 326 F F F 21 420-439 178 178 178 322 322 322 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 182 182 182 326 326 326 T T T 26 520-539 182 182 182 326 326 326 F F F 27 540-559 182 182 182 326 326 326 F F F 28 560-579 176 176 176 318 318 318 F F F 29 580-599 176 176 176 316 316 316 T T T Max gen: 182, Max flush: 326 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 323 323 323 F F F 1 020-039 182 182 182 326 326 326 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 181 181 181 325 325 325 F F F 4 080-099 181 181 181 325 325 325 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 179 179 179 323 323 323 F F F 8 160-179 181 181 181 325 325 325 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 182 182 182 326 326 326 F F F 11 220-239 182 182 182 326 326 326 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 183 183 183 327 327 327 F F F 14 280-299 182 182 182 326 326 326 F F F 15 300-319 183 183 183 327 327 327 F F F 16 320-339 181 181 181 325 325 325 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 183 183 183 327 327 327 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 178 178 178 322 322 322 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 182 182 182 326 326 326 F F F 27 540-559 182 182 182 326 326 326 F F F 28 560-579 176 176 176 318 318 318 F F F 29 580-599 183 183 183 327 327 327 F F F Max gen: 183, Max flush: 327 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.170396465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.21286554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.213449294Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.213480075Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.213494275Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.213508956Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918} {"msg":"Crucible 90aee626-2780-430c-a9ec-93a327b2c800 has session id: 4d80558a-1a28-448e-b64a-bc80e909e497","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.213952226Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Upstairs opts: Upstairs UUID: 90aee626-2780-430c-a9ec-93a327b2c800, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.214000798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Crucible stats registered with UUID: 90aee626-2780-430c-a9ec-93a327b2c800","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.214031678Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243683347Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243716568Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918} {"msg":"90aee626-2780-430c-a9ec-93a327b2c800 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243747429Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243764869Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.24379529Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.24381405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243842361Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243910553Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243948914Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.243982444Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244016365Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244048846Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_connection connected from Ok(127.0.0.1:64303)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244149748Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_connection connected from Ok(127.0.0.1:39055)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244190089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_connection connected from Ok(127.0.0.1:47192)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.24421954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244798714Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244828764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244844225Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244858195Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244872036Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.244890446Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.245500271Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.245531701Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.245549822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246065584Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246095295Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246111385Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246124335Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246138306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246151926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246168196Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246182407Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[0]R flush_numbers[0..12]: [323, 326, 323, 325, 325, 326, 324, 323, 325, 305, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246196587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[0]R generation[0..12]: [179, 182, 179, 181, 181, 182, 180, 179, 181, 172, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246210637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246224688Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[1]R flush_numbers[0..12]: [323, 326, 323, 325, 325, 326, 324, 323, 325, 305, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246238798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[1]R generation[0..12]: [179, 182, 179, 181, 181, 182, 180, 179, 181, 172, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246252798Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246266969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[2]R flush_numbers[0..12]: [323, 326, 323, 325, 325, 326, 324, 323, 325, 305, 326, 326]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246280689Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[2]R generation[0..12]: [179, 182, 179, 181, 181, 182, 180, 179, 181, 172, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246294619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.24630879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.24632217Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246343621Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246356961Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246372011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"":"downstairs","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246385572Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246397722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"0","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246411052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"1","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246424353Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"client":"2","":"downstairs","session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246437693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246449953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"90aee626-2780-430c-a9ec-93a327b2c800 is now active with session: 4d80558a-1a28-448e-b64a-bc80e909e497","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246462874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246475174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918,"session_id":"4d80558a-1a28-448e-b64a-bc80e909e497"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:24.246487864Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 492 len 12288 data: 12 10 13 02/30 Write block 397 len 4096 data: 16 03/30 Write block 488 len 40960 data: 12 12 12 13 13 11 14 13 11 11 04/30 Write block 579 len 36864 data: 16 17 15 14 14 13 11 13 14 05/30 Flush 06/30 Write block 399 len 28672 data: 12 15 17 21 23 19 14 07/30 Flush 08/30 Write block 432 len 8192 data: 14 13 09/30 Write block 15 len 4096 data: 11 10/30 Read block 136 len 24576 11/30 Read block 427 len 12288 12/30 Read block 219 len 40960 13/30 Write block 142 len 24576 data: 16 14 14 11 12 13 14/30 Read block 284 len 20480 15/30 Read block 460 len 40960 16/30 Flush 17/30 Write block 201 len 4096 data: 14 18/30 Read block 552 len 36864 19/30 Read block 452 len 16384 20/30 Write block 481 len 28672 data: 20 18 16 14 14 11 14 21/30 Read block 501 len 4096 22/30 Read block 243 len 28672 23/30 Write block 593 len 4096 data: 11 24/30 Read block 7 len 24576 25/30 Read block 185 len 36864 26/30 Write block 212 len 36864 data: 10 11 8 8 10 10 10 12 14 27/30 Read block 169 len 28672 28/30 Read block 137 len 32768 29/30 Read block 455 len 4096 30/30 Write block 142 len 12288 data: 17 15 15 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 1 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:24.265904498Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 330 330 330 F F F 1 020-039 182 182 182 326 326 326 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 181 181 181 325 325 325 F F F 4 080-099 181 181 181 325 325 325 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 184 184 184 330 330 330 T T T 8 160-179 181 181 181 325 325 325 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 182 182 182 326 326 326 T T T 11 220-239 182 182 182 326 326 326 T T T 12 240-259 178 178 178 321 321 321 F F F 13 260-279 183 183 183 327 327 327 F F F 14 280-299 182 182 182 326 326 326 F F F 15 300-319 183 183 183 327 327 327 F F F 16 320-339 181 181 181 325 325 325 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 184 184 184 329 329 329 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 184 184 184 330 330 330 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 184 184 184 328 328 328 T T T 25 500-519 183 183 183 327 327 327 F F F 26 520-539 182 182 182 326 326 326 F F F 27 540-559 182 182 182 326 326 326 F F F 28 560-579 184 184 184 328 328 328 F F F 29 580-599 184 184 184 328 328 328 T T T Max gen: 184, Max flush: 330 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 330 330 330 F F F 1 020-039 182 182 182 326 326 326 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 181 181 181 325 325 325 F F F 4 080-099 181 181 181 325 325 325 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 185 185 185 331 331 331 F F F 8 160-179 181 181 181 325 325 325 F F F 9 180-199 172 172 172 305 305 305 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 183 183 183 327 327 327 F F F 14 280-299 182 182 182 326 326 326 F F F 15 300-319 183 183 183 327 327 327 F F F 16 320-339 181 181 181 325 325 325 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 184 184 184 329 329 329 F F F 20 400-419 184 184 184 329 329 329 F F F 21 420-439 184 184 184 330 330 330 F F F 22 440-459 175 175 175 314 314 314 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 182 182 182 326 326 326 F F F 27 540-559 182 182 182 326 326 326 F F F 28 560-579 184 184 184 328 328 328 F F F 29 580-599 185 185 185 331 331 331 F F F Max gen: 185, Max flush: 331 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.527663652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.570190398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.570759822Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.570796503Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.570811033Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.570825164Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926} {"msg":"Crucible 5cacc669-91e7-49e2-afe6-e3ad7c75b725 has session id: 41c23f96-b182-49dc-b5ad-a09d2f6d6109","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.571294145Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Upstairs opts: Upstairs UUID: 5cacc669-91e7-49e2-afe6-e3ad7c75b725, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.571328005Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Crucible stats registered with UUID: 5cacc669-91e7-49e2-afe6-e3ad7c75b725","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.571343776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602215724Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602258235Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926} {"msg":"5cacc669-91e7-49e2-afe6-e3ad7c75b725 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602294326Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602314336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602373878Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602411768Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602448889Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.60248525Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602522721Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602557192Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602591953Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602624144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_connection connected from Ok(127.0.0.1:37327)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602660054Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_connection connected from Ok(127.0.0.1:48169)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602694405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_connection connected from Ok(127.0.0.1:34125)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.602729416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.603254479Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.603283399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.60330179Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.60331646Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.603512205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.603539406Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.603975286Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604014897Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604295484Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604522439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.60456254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604591221Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604607731Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604623561Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604637532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604814156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604850447Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[0]R flush_numbers[0..12]: [330, 326, 323, 325, 325, 326, 324, 331, 325, 305, 331, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604880298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[0]R generation[0..12]: [184, 182, 179, 181, 181, 182, 180, 185, 181, 172, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604908138Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.604934559Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[1]R flush_numbers[0..12]: [330, 326, 323, 325, 325, 326, 324, 331, 325, 305, 331, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.6049587Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[1]R generation[0..12]: [184, 182, 179, 181, 181, 182, 180, 185, 181, 172, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.60498465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605012341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[2]R flush_numbers[0..12]: [330, 326, 323, 325, 325, 326, 324, 331, 325, 305, 331, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605038661Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[2]R generation[0..12]: [184, 182, 179, 181, 181, 182, 180, 185, 181, 172, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605065272Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605103543Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605134764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605164114Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605192115Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605220996Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"":"downstairs","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605249106Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605284157Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"0","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605313018Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"1","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605332708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"client":"2","":"downstairs","session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605346969Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605359539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"5cacc669-91e7-49e2-afe6-e3ad7c75b725 is now active with session: 41c23f96-b182-49dc-b5ad-a09d2f6d6109","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.605373189Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.60538548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926,"session_id":"41c23f96-b182-49dc-b5ad-a09d2f6d6109"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:28.60539811Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 16384 data: 15 18 16 16 02/30 Write block 280 len 16384 data: 10 9 9 12 03/30 Write block 554 len 24576 data: 16 14 14 19 15 11 04/30 Write block 60 len 24576 data: 14 17 16 14 16 19 05/30 Write block 277 len 12288 data: 10 8 8 06/30 Write block 413 len 12288 data: 19 17 17 07/30 Write block 532 len 24576 data: 13 12 11 13 14 17 08/30 Write block 597 len 8192 data: 9 7 09/30 Read block 513 len 36864 10/30 Write block 576 len 16384 data: 17 15 17 17 11/30 Read block 567 len 36864 12/30 Write block 335 len 4096 data: 14 13/30 Flush 14/30 Write block 336 len 36864 data: 10 11 13 12 14 12 13 12 15 15/30 Write block 294 len 20480 data: 20 21 21 22 17 16/30 Write block 160 len 4096 data: 22 17/30 Write block 1 len 4096 data: 6 18/30 Write block 384 len 40960 data: 18 15 16 19 14 16 15 14 14 14 19/30 Read block 438 len 20480 20/30 Write block 418 len 28672 data: 13 13 12 14 16 16 15 21/30 Write block 451 len 32768 data: 12 11 11 12 10 13 11 12 22/30 Write block 298 len 36864 data: 18 12 10 10 8 10 8 9 14 23/30 Read block 432 len 8192 24/30 Read block 537 len 8192 25/30 Read block 267 len 4096 26/30 Flush 27/30 Write block 24 len 28672 data: 16 18 18 14 14 12 15 28/30 Read block 137 len 8192 29/30 Write block 381 len 8192 data: 16 18 30/30 Write block 212 len 16384 data: 11 12 9 9 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 2 3 2 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:28.621753231Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 182 182 182 326 326 326 T T T 2 040-059 179 179 179 323 323 323 F F F 3 060-079 186 186 186 332 332 332 F F F 4 080-099 181 181 181 325 325 325 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 185 185 185 331 331 331 F F F 8 160-179 186 186 186 333 333 333 F F F 9 180-199 186 186 186 332 332 332 F F F 10 200-219 185 185 185 331 331 331 T T T 11 220-239 185 185 185 331 331 331 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 186 186 186 332 332 332 F F F 14 280-299 186 186 186 333 333 333 F F F 15 300-319 186 186 186 333 333 333 F F F 16 320-339 186 186 186 333 333 333 F F F 17 340-359 186 186 186 333 333 333 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 186 186 186 333 333 333 T T T 20 400-419 186 186 186 333 333 333 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 333 333 333 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 186 186 186 332 332 332 F F F 27 540-559 186 186 186 332 332 332 F F F 28 560-579 186 186 186 332 332 332 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 186, Max flush: 333 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 186 186 186 332 332 332 F F F 4 080-099 181 181 181 325 325 325 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 185 185 185 331 331 331 F F F 8 160-179 186 186 186 333 333 333 F F F 9 180-199 186 186 186 332 332 332 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 186 186 186 332 332 332 F F F 14 280-299 186 186 186 333 333 333 F F F 15 300-319 186 186 186 333 333 333 F F F 16 320-339 186 186 186 333 333 333 F F F 17 340-359 186 186 186 333 333 333 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 187 187 187 334 334 334 F F F 20 400-419 186 186 186 333 333 333 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 333 333 333 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 186 186 186 332 332 332 F F F 27 540-559 186 186 186 332 332 332 F F F 28 560-579 186 186 186 332 332 332 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 187, Max flush: 334 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.881733671Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:32.924173285Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.924550934Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.955870483Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.956458917Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.956494708Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.956510658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.956525469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934} {"msg":"Crucible 9c0c7a2f-6879-4239-9c77-cacb21a34158 has session id: da9e2990-cdbb-452c-affb-d18114286751","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.956924198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Upstairs opts: Upstairs UUID: 9c0c7a2f-6879-4239-9c77-cacb21a34158, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.956956799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Crucible stats registered with UUID: 9c0c7a2f-6879-4239-9c77-cacb21a34158","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.956973119Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987256603Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987282314Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934} {"msg":"9c0c7a2f-6879-4239-9c77-cacb21a34158 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987359156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987382396Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987407867Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987423777Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987438078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987460528Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987475648Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987498619Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.98752497Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.98755108Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_connection connected from Ok(127.0.0.1:45371)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987583451Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_connection connected from Ok(127.0.0.1:56747)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987611902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_connection connected from Ok(127.0.0.1:47725)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.987640363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.988327759Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.98836918Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.988456652Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.988490653Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.988507383Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.988521713Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.988971764Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989010905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989152049Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989538868Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989569209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989584529Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.98964592Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989676011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989692341Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989705472Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989719222Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[0]R flush_numbers[0..12]: [333, 334, 323, 332, 325, 326, 324, 331, 333, 332, 334, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989733422Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[0]R generation[0..12]: [186, 187, 179, 186, 181, 182, 180, 185, 186, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989752053Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989767233Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[1]R flush_numbers[0..12]: [333, 334, 323, 332, 325, 326, 324, 331, 333, 332, 334, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989781954Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[1]R generation[0..12]: [186, 187, 179, 186, 181, 182, 180, 185, 186, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989796254Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989810614Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[2]R flush_numbers[0..12]: [333, 334, 323, 332, 325, 326, 324, 331, 333, 332, 334, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989824555Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[2]R generation[0..12]: [186, 187, 179, 186, 181, 182, 180, 185, 186, 186, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989838435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989852565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989865806Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989879276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989892536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"All extents matchWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989913397Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"":"downstairs","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989926727Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989938838Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"0","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989952288Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"1","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989969758Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"client":"2","":"downstairs","session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989983888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.989996549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"9c0c7a2f-6879-4239-9c77-cacb21a34158 is now active with session: da9e2990-cdbb-452c-affb-d18114286751","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.990009439Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.990022589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934,"session_id":"da9e2990-cdbb-452c-affb-d18114286751"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:32.99003515Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 562 len 36864 data: 9 10 8 8 7 8 8 11 13 02/30 Write block 136 len 24576 data: 11 9 12 13 16 19 03/30 Read block 322 len 16384 04/30 Write block 405 len 4096 data: 15 05/30 Read block 492 len 12288 06/30 Read block 184 len 40960 07/30 Read block 117 len 4096 08/30 Read block 3 len 16384 09/30 Write block 593 len 20480 data: 12 11 10 10 10 10/30 Write block 402 len 12288 data: 22 24 20 11/30 Read block 172 len 12288 12/30 Write block 75 len 36864 data: 9 10 16 16 12 8 12 11 10 13/30 Read block 469 len 8192 14/30 Read block 476 len 40960 15/30 Read block 572 len 12288 16/30 Read block 367 len 32768 17/30 Read block 258 len 40960 18/30 Read block 481 len 28672 19/30 Read block 583 len 40960 20/30 Write block 169 len 16384 data: 9 8 8 10 21/30 Write block 202 len 32768 data: 12 13 9 12 11 12 13 12 22/30 Write block 128 len 24576 data: 11 11 11 9 10 9 23/30 Read block 537 len 4096 24/30 Read block 341 len 20480 25/30 Read block 104 len 32768 26/30 Write block 382 len 20480 data: 19 17 19 16 17 27/30 Read block 346 len 20480 28/30 Write block 571 len 36864 data: 17 17 15 18 19 18 16 18 18 29/30 Write block 77 len 24576 data: 17 17 13 9 13 12 30/30 Read block 83 len 28672 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 29 30 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:33.010372826Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 186 186 186 332 332 332 T T T 4 080-099 181 181 181 325 325 325 T T T 5 100-119 182 182 182 326 326 326 F F F 6 120-139 180 180 180 324 324 324 T T T 7 140-159 185 185 185 331 331 331 T T T 8 160-179 186 186 186 333 333 333 T T T 9 180-199 186 186 186 332 332 332 F F F 10 200-219 187 187 187 334 334 334 T T T 11 220-239 185 185 185 331 331 331 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 186 186 186 332 332 332 F F F 14 280-299 186 186 186 333 333 333 F F F 15 300-319 186 186 186 333 333 333 F F F 16 320-339 186 186 186 333 333 333 F F F 17 340-359 186 186 186 333 333 333 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 187 187 187 334 334 334 T T T 20 400-419 186 186 186 333 333 333 T T T 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 333 333 333 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 186 186 186 332 332 332 F F F 27 540-559 186 186 186 332 332 332 F F F 28 560-579 186 186 186 332 332 332 T T T 29 580-599 186 186 186 332 332 332 T T T Max gen: 187, Max flush: 334 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 188 188 188 335 335 335 F F F 4 080-099 188 188 188 335 335 335 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 188 188 188 335 335 335 F F F 7 140-159 188 188 188 335 335 335 F F F 8 160-179 188 188 188 335 335 335 F F F 9 180-199 186 186 186 332 332 332 F F F 10 200-219 188 188 188 335 335 335 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 186 186 186 332 332 332 F F F 14 280-299 186 186 186 333 333 333 F F F 15 300-319 186 186 186 333 333 333 F F F 16 320-339 186 186 186 333 333 333 F F F 17 340-359 186 186 186 333 333 333 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 188 188 188 335 335 335 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 186 186 186 333 333 333 F F F 22 440-459 186 186 186 333 333 333 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 186 186 186 332 332 332 F F F 27 540-559 186 186 186 332 332 332 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 188, Max flush: 335 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.330571176Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:37.372069478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.372452187Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.402762152Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.403313025Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.403347926Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.403362316Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.403376977Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942} {"msg":"Crucible fd838b8f-4420-450c-89e4-cbfb1c2ccf18 has session id: 28454fff-a6ef-427e-8924-0c8dc94fc94b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.403846098Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Upstairs opts: Upstairs UUID: fd838b8f-4420-450c-89e4-cbfb1c2ccf18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.403890219Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Crucible stats registered with UUID: fd838b8f-4420-450c-89e4-cbfb1c2ccf18","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.40391211Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433377224Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433408405Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942} {"msg":"fd838b8f-4420-450c-89e4-cbfb1c2ccf18 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433477326Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433498987Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433539338Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433574868Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.43361026Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.43364337Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433676981Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433711382Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433744473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433779473Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44957)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433819334Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63665)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.433947837Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40563)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.434009129Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.434511321Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.434538872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.434554952Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.434569592Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.434583453Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.434598693Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435098635Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435213328Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435237428Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435649128Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435678269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435693249Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435765371Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435784831Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435798622Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435815722Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435830062Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[0]R flush_numbers[0..12]: [333, 334, 323, 335, 335, 326, 335, 335, 335, 332, 335, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435844263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[0]R generation[0..12]: [186, 187, 179, 188, 188, 182, 188, 188, 188, 186, 188, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435858223Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435872343Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[1]R flush_numbers[0..12]: [333, 334, 323, 335, 335, 326, 335, 335, 335, 332, 335, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435886084Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[1]R generation[0..12]: [186, 187, 179, 188, 188, 182, 188, 188, 188, 186, 188, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435899854Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435913944Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[2]R flush_numbers[0..12]: [333, 334, 323, 335, 335, 326, 335, 335, 335, 332, 335, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435927885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[2]R generation[0..12]: [186, 187, 179, 188, 188, 182, 188, 188, 188, 186, 188, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435941795Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435955905Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435969046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435982536Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.435995697Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436018717Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"":"downstairs","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436032637Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436044968Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"0","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436058478Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"1","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436071718Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"client":"2","":"downstairs","session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436085139Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436097539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"fd838b8f-4420-450c-89e4-cbfb1c2ccf18 is now active with session: 28454fff-a6ef-427e-8924-0c8dc94fc94b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436110379Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.436122469Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942,"session_id":"28454fff-a6ef-427e-8924-0c8dc94fc94b"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:37.4361348Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 12288 data: 17 15 15 02/30 Write block 320 len 12288 data: 15 16 13 03/30 Read block 237 len 20480 04/30 Read block 40 len 24576 05/30 Read block 204 len 20480 06/30 Read block 88 len 28672 07/30 Write block 255 len 28672 data: 14 13 13 15 15 14 12 08/30 Read block 582 len 40960 09/30 Read block 465 len 20480 10/30 Read block 470 len 16384 11/30 Read block 59 len 24576 12/30 Write block 165 len 32768 data: 17 16 10 10 10 9 9 11 13/30 Read block 536 len 40960 14/30 Read block 414 len 8192 15/30 Read block 224 len 40960 16/30 Read block 438 len 16384 17/30 Write block 544 len 40960 data: 17 14 12 12 13 11 13 13 17 15 18/30 Write block 526 len 4096 data: 6 19/30 Write block 157 len 4096 data: 22 20/30 Write block 549 len 32768 data: 12 14 14 18 16 17 15 15 21/30 Read block 322 len 20480 22/30 Read block 504 len 32768 23/30 Read block 554 len 24576 24/30 Write block 384 len 40960 data: 20 17 18 20 15 17 16 15 15 15 25/30 Read block 77 len 8192 26/30 Write block 341 len 28672 data: 13 14 13 16 16 20 21 27/30 Write block 428 len 24576 data: 19 18 18 16 16 14 28/30 Write block 315 len 40960 data: 19 19 18 19 16 16 17 14 14 14 29/30 Read block 347 len 32768 30/30 Read block 516 len 28672 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:37.456690171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 188 188 188 335 335 335 F F F 4 080-099 188 188 188 335 335 335 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 188 188 188 335 335 335 F F F 7 140-159 188 188 188 335 335 335 T T T 8 160-179 188 188 188 335 335 335 T T T 9 180-199 186 186 186 332 332 332 F F F 10 200-219 188 188 188 335 335 335 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 178 178 178 321 321 321 T T T 13 260-279 186 186 186 332 332 332 T T T 14 280-299 186 186 186 333 333 333 F F F 15 300-319 186 186 186 333 333 333 T T T 16 320-339 186 186 186 333 333 333 T T T 17 340-359 186 186 186 333 333 333 T T T 18 360-379 182 182 182 326 326 326 F F F 19 380-399 188 188 188 335 335 335 T T T 20 400-419 188 188 188 335 335 335 F F F 21 420-439 186 186 186 333 333 333 T T T 22 440-459 186 186 186 333 333 333 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 186 186 186 332 332 332 T T T 27 540-559 186 186 186 332 332 332 T T T 28 560-579 188 188 188 335 335 335 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 188, Max flush: 335 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 F F F 2 040-059 179 179 179 323 323 323 F F F 3 060-079 188 188 188 335 335 335 F F F 4 080-099 188 188 188 335 335 335 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 188 188 188 335 335 335 F F F 7 140-159 189 189 189 336 336 336 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 186 186 186 332 332 332 F F F 10 200-219 188 188 188 335 335 335 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 186 186 186 333 333 333 F F F 15 300-319 189 189 189 336 336 336 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 189 189 189 336 336 336 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 186 186 186 333 333 333 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 183 183 183 327 327 327 F F F 26 520-539 189 189 189 336 336 336 F F F 27 540-559 189 189 189 336 336 336 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 189, Max flush: 336 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.790805903Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.833574526Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.83416719Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.834210451Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.834228781Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.834246872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950} {"msg":"Crucible df1fbc81-0bf2-4d78-9c6e-6741bc3dbc14 has session id: 311d5ead-9208-4dd9-b170-2bb0fbd621a2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.834655011Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Upstairs opts: Upstairs UUID: df1fbc81-0bf2-4d78-9c6e-6741bc3dbc14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.834704363Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Crucible stats registered with UUID: df1fbc81-0bf2-4d78-9c6e-6741bc3dbc14","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.834729603Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864221658Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864249799Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950} {"msg":"df1fbc81-0bf2-4d78-9c6e-6741bc3dbc14 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864324551Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864343491Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864368152Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864382902Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864407603Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864434623Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864457994Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864479504Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864506425Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864533916Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63498)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864568526Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58561)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864847273Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38665)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.864883924Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.865279834Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.865306594Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.865438647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.865483078Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.865513239Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.86553066Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.865941739Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.86597885Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866249697Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866467112Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866496203Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866512263Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866533233Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866548194Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866566014Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866737548Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866758029Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[0]R flush_numbers[0..12]: [333, 334, 323, 335, 335, 326, 335, 336, 336, 332, 335, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866772549Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[0]R generation[0..12]: [186, 187, 179, 188, 188, 182, 188, 189, 189, 186, 188, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.86678817Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.86680269Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[1]R flush_numbers[0..12]: [333, 334, 323, 335, 335, 326, 335, 336, 336, 332, 335, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.86681751Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[1]R generation[0..12]: [186, 187, 179, 188, 188, 182, 188, 189, 189, 186, 188, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866832171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866847081Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[2]R flush_numbers[0..12]: [333, 334, 323, 335, 335, 326, 335, 336, 336, 332, 335, 331]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866861171Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[2]R generation[0..12]: [186, 187, 179, 188, 188, 182, 188, 189, 189, 186, 188, 185]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866875012Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866889632Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866902942Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Generation requested: 192 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866917153Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866933943Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"All extents match",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866956874Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"":"downstairs","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866970614Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866983474Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"0","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.866997964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"1","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.867012345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"client":"2","":"downstairs","session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.867026435Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.867039565Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"df1fbc81-0bf2-4d78-9c6e-6741bc3dbc14 is now active with session: 311d5ead-9208-4dd9-b170-2bb0fbd621a2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.867052556Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.867064736Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950,"session_id":"311d5ead-9208-4dd9-b170-2bb0fbd621a2"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:41.867077757Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 200 len 12288 data: 14 15 13 02/30 Read block 495 len 28672 03/30 Read block 220 len 8192 04/30 Write block 123 len 28672 data: 16 15 16 15 15 12 12 05/30 Write block 377 len 32768 data: 14 17 17 14 17 20 18 21 06/30 Flush 07/30 Write block 444 len 16384 data: 12 11 12 10 08/30 Write block 492 len 40960 data: 14 12 15 14 12 12 10 12 13 17 09/30 Read block 254 len 20480 10/30 Write block 590 len 20480 data: 14 14 12 13 12 11/30 Write block 222 len 12288 data: 16 16 16 12/30 Flush 13/30 Write block 517 len 32768 data: 15 12 9 10 9 10 10 9 14/30 Write block 148 len 28672 data: 11 11 11 10 13 17 20 15/30 Flush 16/30 Write block 471 len 40960 data: 11 9 10 6 6 9 8 11 14 17 17/30 Write block 513 len 36864 data: 14 12 18 15 16 13 10 11 10 18/30 Read block 162 len 20480 19/30 Write block 218 len 36864 data: 11 13 15 14 17 17 17 19 17 20/30 Read block 411 len 4096 21/30 Read block 203 len 20480 22/30 Read block 253 len 32768 23/30 Read block 103 len 36864 24/30 Read block 398 len 16384 25/30 Flush 26/30 Write block 128 len 12288 data: 13 13 12 27/30 Read block 169 len 28672 28/30 Write block 50 len 36864 data: 20 14 15 14 14 17 16 13 14 29/30 Write block 113 len 40960 data: 10 9 9 13 14 10 15 16 15 15 30/30 Write block 522 len 16384 data: 11 11 10 9 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:41.887458114Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 F F F 2 040-059 179 179 179 323 323 323 T T T 3 060-079 188 188 188 335 335 335 F F F 4 080-099 188 188 188 335 335 335 F F F 5 100-119 182 182 182 326 326 326 T T T 6 120-139 192 192 192 337 337 337 T T T 7 140-159 192 192 192 339 339 339 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 186 186 186 332 332 332 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 192 192 192 340 340 340 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 186 186 186 333 333 333 F F F 15 300-319 189 189 189 336 336 336 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 192 192 192 337 337 337 F F F 19 380-399 192 192 192 337 337 337 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 192 192 192 338 338 338 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 192 192 192 340 340 340 F F F 26 520-539 192 192 192 340 340 340 T T T 27 540-559 189 189 189 336 336 336 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 192 192 192 338 338 338 F F F Max gen: 192, Max flush: 340 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 F F F 2 040-059 193 193 193 341 341 341 F F F 3 060-079 188 188 188 335 335 335 F F F 4 080-099 188 188 188 335 335 335 F F F 5 100-119 193 193 193 341 341 341 F F F 6 120-139 193 193 193 341 341 341 F F F 7 140-159 192 192 192 339 339 339 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 186 186 186 332 332 332 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 192 192 192 340 340 340 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 186 186 186 333 333 333 F F F 15 300-319 189 189 189 336 336 336 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 192 192 192 337 337 337 F F F 19 380-399 192 192 192 337 337 337 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 192 192 192 338 338 338 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 192 192 192 340 340 340 F F F 26 520-539 193 193 193 341 341 341 F F F 27 540-559 189 189 189 336 336 336 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 192 192 192 338 338 338 F F F Max gen: 193, Max flush: 341 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.141772399Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.185195327Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.185838352Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.185888723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.185912674Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.185940945Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958} {"msg":"Crucible 4d97fecd-9345-4606-8861-06a292062b0b has session id: fdae254c-f637-4ac5-ad89-1235c31a242d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.186340014Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Upstairs opts: Upstairs UUID: 4d97fecd-9345-4606-8861-06a292062b0b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.186388315Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Crucible stats registered with UUID: 4d97fecd-9345-4606-8861-06a292062b0b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.186410306Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.21586236Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.215906131Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958} {"msg":"4d97fecd-9345-4606-8861-06a292062b0b active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.215989863Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216019024Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216044524Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216059895Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216073925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216087465Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216101046Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216116956Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216141877Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216167347Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38362)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216193398Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56306)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216433534Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54284)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216472255Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216951456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.216990047Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.217012527Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.217035738Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.217050519Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.217065289Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.217562891Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.217714424Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.217744205Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218052572Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218094073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218125054Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218250357Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218271608Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218285928Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218298388Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218312029Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[0]R flush_numbers[0..12]: [333, 334, 341, 335, 335, 341, 341, 339, 336, 332, 340, 340]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218330329Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[0]R generation[0..12]: [186, 187, 193, 188, 188, 193, 193, 192, 189, 186, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.21834489Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.2183601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[1]R flush_numbers[0..12]: [333, 334, 341, 335, 335, 341, 341, 339, 336, 332, 340, 340]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.21837466Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[1]R generation[0..12]: [186, 187, 193, 188, 188, 193, 193, 192, 189, 186, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218388671Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218402951Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[2]R flush_numbers[0..12]: [333, 334, 341, 335, 335, 341, 341, 339, 336, 332, 340, 340]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218416821Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[2]R generation[0..12]: [186, 187, 193, 188, 188, 193, 193, 192, 189, 186, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218430612Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218444662Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218457672Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218470893Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218483843Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:14:46.218496673Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"":"downstairs","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218518003Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218532144Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"0","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218545564Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"1","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218558775Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"client":"2","":"downstairs","session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218572145Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218584655Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"4d97fecd-9345-4606-8861-06a292062b0b is now active with session: fdae254c-f637-4ac5-ad89-1235c31a242d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218597276Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218609336Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958,"session_id":"fdae254c-f637-4ac5-ad89-1235c31a242d"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:46.218621656Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 24576 data: 18 18 17 16 16 16 02/30 Read block 274 len 20480 03/30 Read block 165 len 8192 04/30 Read block 158 len 28672 05/30 Read block 129 len 24576 06/30 Read block 557 len 36864 07/30 Read block 108 len 36864 08/30 Read block 485 len 4096 09/30 Write block 367 len 24576 data: 17 18 18 13 13 13 10/30 Read block 11 len 32768 11/30 Write block 86 len 4096 data: 7 12/30 Write block 230 len 20480 data: 12 10 12 15 12 13/30 Read block 546 len 12288 14/30 Read block 274 len 36864 15/30 Flush 16/30 Flush 17/30 Write block 543 len 32768 data: 17 18 15 13 13 14 13 15 18/30 Read block 449 len 36864 19/30 Flush 20/30 Write block 298 len 36864 data: 19 13 11 11 9 11 9 10 15 21/30 Write block 152 len 12288 data: 14 18 21 22/30 Flush 23/30 Flush 24/30 Write block 249 len 20480 data: 8 8 10 12 13 25/30 Write block 252 len 12288 data: 13 14 14 26/30 Write block 189 len 16384 data: 9 9 10 10 27/30 Write block 28 len 8192 data: 15 13 28/30 Write block 185 len 8192 data: 13 14 29/30 Read block 105 len 20480 30/30 Write block 516 len 16384 data: 16 17 14 11 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:46.237869416Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 187 187 187 334 334 334 T T T 2 040-059 193 193 193 341 341 341 F F F 3 060-079 188 188 188 335 335 335 F F F 4 080-099 194 194 194 342 342 342 F F F 5 100-119 193 193 193 341 341 341 F F F 6 120-139 193 193 193 341 341 341 F F F 7 140-159 194 194 194 345 345 345 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 186 186 186 332 332 332 T T T 10 200-219 192 192 192 340 340 340 F F F 11 220-239 194 194 194 342 342 342 F F F 12 240-259 189 189 189 336 336 336 T T T 13 260-279 189 189 189 336 336 336 F F F 14 280-299 194 194 194 345 345 345 F F F 15 300-319 194 194 194 345 345 345 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 194 194 194 342 342 342 F F F 19 380-399 192 192 192 337 337 337 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 192 192 192 338 338 338 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 192 192 192 340 340 340 T T T 26 520-539 193 193 193 341 341 341 F F F 27 540-559 194 194 194 344 344 344 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 192 192 192 338 338 338 F F F Max gen: 194, Max flush: 345 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 195 195 195 346 346 346 F F F 2 040-059 193 193 193 341 341 341 F F F 3 060-079 188 188 188 335 335 335 F F F 4 080-099 194 194 194 342 342 342 F F F 5 100-119 193 193 193 341 341 341 F F F 6 120-139 193 193 193 341 341 341 F F F 7 140-159 194 194 194 345 345 345 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 195 195 195 346 346 346 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 194 194 194 342 342 342 F F F 12 240-259 195 195 195 346 346 346 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 194 194 194 345 345 345 F F F 15 300-319 194 194 194 345 345 345 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 194 194 194 342 342 342 F F F 19 380-399 192 192 192 337 337 337 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 192 192 192 338 338 338 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 195 195 195 346 346 346 F F F 26 520-539 193 193 193 341 341 341 F F F 27 540-559 194 194 194 344 344 344 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 192 192 192 338 338 338 F F F Max gen: 195, Max flush: 346 Begin loop 98 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.493045302Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.536315836Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.536907771Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.536946872Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.536962052Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.536976412Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966} {"msg":"Crucible abc9e994-5c0a-4704-bdae-3191e1da494f has session id: 375cc677-38d9-47dd-93ec-46e2d288499f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.537371092Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Upstairs opts: Upstairs UUID: abc9e994-5c0a-4704-bdae-3191e1da494f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.537433433Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Crucible stats registered with UUID: abc9e994-5c0a-4704-bdae-3191e1da494f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.537458174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567033061Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567060471Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966} {"msg":"abc9e994-5c0a-4704-bdae-3191e1da494f active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567145153Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567163684Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567187925Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567207985Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567223115Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567236546Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567250126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567263456Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567278487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567303577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37434)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567345848Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62152)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.5674119Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38057)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.567443521Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568157068Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568187618Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568203879Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568218139Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568356233Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568390523Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568925386Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568956297Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.568973017Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.56951037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569536001Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569550601Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569563181Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569576632Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569590142Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569602552Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569616143Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[0]R flush_numbers[0..12]: [333, 346, 341, 335, 342, 341, 341, 345, 336, 346, 340, 342]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569630073Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[0]R generation[0..12]: [186, 195, 193, 188, 194, 193, 193, 194, 189, 195, 192, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569644293Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569658433Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[1]R flush_numbers[0..12]: [333, 346, 341, 335, 342, 341, 341, 345, 336, 346, 340, 342]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569676564Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[1]R generation[0..12]: [186, 195, 193, 188, 194, 193, 193, 194, 189, 195, 192, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569690964Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569705325Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[2]R flush_numbers[0..12]: [333, 346, 341, 335, 342, 341, 341, 345, 336, 346, 340, 342]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569719345Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[2]R generation[0..12]: [186, 195, 193, 188, 194, 193, 193, 194, 189, 195, 192, 194]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569733035Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-12-09T15:14:50.569747076Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569767856Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569781256Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Next flush: 347","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569794417Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569807647Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"":"downstairs","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569820948Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569833198Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"0","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569846718Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"1","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569860009Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"client":"2","":"downstairs","session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569873339Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569888199Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"abc9e994-5c0a-4704-bdae-3191e1da494f is now active with session: 375cc677-38d9-47dd-93ec-46e2d288499f","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.569901509Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.56991367Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966,"session_id":"375cc677-38d9-47dd-93ec-46e2d288499f"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:50.5699262Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 190 len 28672 data: 10 11 11 12 13 15 16 02/30 Write block 456 len 8192 data: 14 12 03/30 Read block 365 len 32768 04/30 Write block 504 len 20480 data: 20 17 20 21 21 05/30 Write block 369 len 24576 data: 19 14 14 14 11 9 06/30 Read block 122 len 36864 07/30 Read block 174 len 12288 08/30 Write block 530 len 32768 data: 9 11 14 13 12 14 15 18 09/30 Write block 374 len 12288 data: 10 9 11 10/30 Read block 355 len 12288 11/30 Write block 173 len 28672 data: 15 14 13 14 11 14 14 12/30 Write block 227 len 40960 data: 15 12 12 13 11 13 16 13 12 13 13/30 Read block 549 len 36864 14/30 Write block 46 len 24576 data: 18 15 14 16 21 15 15/30 Read block 546 len 24576 16/30 Write block 228 len 24576 data: 13 13 14 12 14 17 17/30 Read block 545 len 40960 18/30 Write block 577 len 8192 data: 17 19 19/30 Read block 561 len 16384 20/30 Write block 268 len 36864 data: 19 23 21 16 19 19 16 16 11 21/30 Read block 564 len 32768 22/30 Read block 547 len 40960 23/30 Read block 419 len 36864 24/30 Read block 43 len 40960 25/30 Flush 26/30 Write block 554 len 12288 data: 18 16 16 27/30 Write block 533 len 40960 data: 14 13 15 16 19 18 18 21 17 17 28/30 Read block 512 len 20480 29/30 Write block 279 len 4096 data: 9 30/30 Write block 60 len 32768 data: 15 18 17 15 17 20 20 17 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:50.590674286Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 195 195 195 346 346 346 F F F 2 040-059 196 196 196 347 347 347 F F F 3 060-079 188 188 188 335 335 335 T T T 4 080-099 194 194 194 342 342 342 F F F 5 100-119 193 193 193 341 341 341 F F F 6 120-139 193 193 193 341 341 341 F F F 7 140-159 194 194 194 345 345 345 F F F 8 160-179 196 196 196 347 347 347 F F F 9 180-199 196 196 196 347 347 347 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 196 196 196 347 347 347 F F F 12 240-259 195 195 195 346 346 346 F F F 13 260-279 196 196 196 347 347 347 T T T 14 280-299 194 194 194 345 345 345 F F F 15 300-319 194 194 194 345 345 345 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 196 196 196 347 347 347 F F F 19 380-399 192 192 192 337 337 337 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 196 196 196 347 347 347 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 196 196 196 347 347 347 F F F 26 520-539 196 196 196 347 347 347 T T T 27 540-559 194 194 194 344 344 344 T T T 28 560-579 196 196 196 347 347 347 F F F 29 580-599 192 192 192 338 338 338 F F F Max gen: 196, Max flush: 347 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 333 333 333 F F F 1 020-039 195 195 195 346 346 346 F F F 2 040-059 196 196 196 347 347 347 F F F 3 060-079 197 197 197 348 348 348 F F F 4 080-099 194 194 194 342 342 342 F F F 5 100-119 193 193 193 341 341 341 F F F 6 120-139 193 193 193 341 341 341 F F F 7 140-159 194 194 194 345 345 345 F F F 8 160-179 196 196 196 347 347 347 F F F 9 180-199 196 196 196 347 347 347 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 196 196 196 347 347 347 F F F 12 240-259 195 195 195 346 346 346 F F F 13 260-279 197 197 197 348 348 348 F F F 14 280-299 194 194 194 345 345 345 F F F 15 300-319 194 194 194 345 345 345 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 196 196 196 347 347 347 F F F 19 380-399 192 192 192 337 337 337 F F F 20 400-419 188 188 188 335 335 335 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 196 196 196 347 347 347 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 196 196 196 347 347 347 F F F 26 520-539 197 197 197 348 348 348 F F F 27 540-559 197 197 197 348 348 348 F F F 28 560-579 196 196 196 347 347 347 F F F 29 580-599 192 192 192 338 338 338 F F F Max gen: 197, Max flush: 348 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.845152324Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:54.886505723Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.886882072Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.917513334Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.918131259Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.91817539Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.91819476Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.918212731Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974} {"msg":"Crucible 6e0cdfef-34b9-4d87-9345-1698189cac67 has session id: 54f6b8ff-0c51-4251-81b6-7049f7667068","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.918576569Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Upstairs opts: Upstairs UUID: 6e0cdfef-34b9-4d87-9345-1698189cac67, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.91861835Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Crucible stats registered with UUID: 6e0cdfef-34b9-4d87-9345-1698189cac67","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.918637061Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948225308Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948256089Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974} {"msg":"6e0cdfef-34b9-4d87-9345-1698189cac67 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948330481Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948354161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948382282Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948400853Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948417993Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948439574Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948457274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948483404Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948517595Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948549986Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_connection connected from Ok(127.0.0.1:37832)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948585457Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_connection connected from Ok(127.0.0.1:37388)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948617888Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_connection connected from Ok(127.0.0.1:38550)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.948646498Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.949419487Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.949449118Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.949465468Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.949480438Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.949502689Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.949530129Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950084083Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950131554Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950165575Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950685517Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950718538Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950734668Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950747639Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950761589Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950775209Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950787679Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.95080139Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[0]R flush_numbers[0..12]: [333, 346, 347, 348, 342, 341, 341, 345, 347, 347, 340, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.9508156Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[0]R generation[0..12]: [186, 195, 196, 197, 194, 193, 193, 194, 196, 196, 192, 196]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.9508298Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950844161Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[1]R flush_numbers[0..12]: [333, 346, 347, 348, 342, 341, 341, 345, 347, 347, 340, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950858271Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[1]R generation[0..12]: [186, 195, 196, 197, 194, 193, 193, 194, 196, 196, 192, 196]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950872261Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950889912Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[2]R flush_numbers[0..12]: [333, 346, 347, 348, 342, 341, 341, 345, 347, 347, 340, 347]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950904532Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[2]R generation[0..12]: [186, 195, 196, 197, 194, 193, 193, 194, 196, 196, 192, 196]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950918423Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950932463Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950945613Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950959174Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-12-09T15:14:54.950972414Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.950992535Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"":"downstairs","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951006015Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951018355Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"0","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951031845Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"1","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951045346Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"client":"2","":"downstairs","session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951058776Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951071126Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"6e0cdfef-34b9-4d87-9345-1698189cac67 is now active with session: 54f6b8ff-0c51-4251-81b6-7049f7667068","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951084037Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951098577Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974,"session_id":"54f6b8ff-0c51-4251-81b6-7049f7667068"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:14:54.951111347Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 190 len 28672 data: 11 12 12 13 14 16 17 03/30 Write block 390 len 16384 data: 17 16 16 16 04/30 Read block 50 len 20480 05/30 Read block 28 len 32768 06/30 Flush 07/30 Write block 556 len 40960 data: 17 20 16 12 9 7 10 11 9 9 08/30 Write block 395 len 24576 data: 15 16 17 15 13 16 09/30 Write block 12 len 8192 data: 11 11 10/30 Read block 64 len 40960 11/30 Write block 138 len 20480 data: 13 14 17 20 18 12/30 Flush 13/30 Write block 374 len 32768 data: 11 10 12 15 18 18 15 18 14/30 Flush 15/30 Write block 161 len 32768 data: 21 23 17 17 18 17 11 11 16/30 Read block 77 len 12288 17/30 Write block 142 len 4096 data: 19 18/30 Read block 468 len 24576 19/30 Read block 426 len 16384 20/30 Read block 490 len 40960 21/30 Write block 78 len 28672 data: 18 14 10 14 13 11 9 22/30 Write block 547 len 28672 data: 14 15 14 16 15 19 17 23/30 Write block 596 len 16384 data: 11 11 8 3 24/30 Write block 385 len 4096 data: 18 25/30 Read block 410 len 36864 26/30 Flush 27/30 Write block 175 len 4096 data: 14 28/30 Write block 64 len 36864 data: 18 21 21 18 14 13 13 12 10 29/30 Write block 550 len 16384 data: 17 16 20 18 30/30 Read block 166 len 4096 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 9 Done Sent Sent false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 3 3 6 Done 4 1 1 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:14:54.969810274Z","hostname":"w-01JENZACV33S8546AT9Y1F9JXE","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 351 351 351 F F F 1 020-039 195 195 195 346 346 346 F F F 2 040-059 196 196 196 347 347 347 F F F 3 060-079 198 198 198 353 353 353 T T T 4 080-099 198 198 198 353 353 353 F F F 5 100-119 193 193 193 341 341 341 F F F 6 120-139 198 198 198 351 351 351 F F F 7 140-159 198 198 198 353 353 353 F F F 8 160-179 198 198 198 353 353 353 T T T 9 180-199 198 198 198 350 350 350 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 196 196 196 347 347 347 F F F 12 240-259 195 195 195 346 346 346 F F F 13 260-279 197 197 197 348 348 348 F F F 14 280-299 194 194 194 345 345 345 F F F 15 300-319 194 194 194 345 345 345 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 198 198 198 352 352 352 F F F 19 380-399 198 198 198 353 353 353 F F F 20 400-419 198 198 198 351 351 351 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 196 196 196 347 347 347 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 196 196 196 347 347 347 F F F 26 520-539 197 197 197 348 348 348 F F F 27 540-559 198 198 198 353 353 353 T T T 28 560-579 198 198 198 351 351 351 F F F 29 580-599 198 198 198 353 353 353 F F F Max gen: 198, Max flush: 353 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 351 351 351 F F F 1 020-039 195 195 195 346 346 346 F F F 2 040-059 196 196 196 347 347 347 F F F 3 060-079 199 199 199 354 354 354 F F F 4 080-099 198 198 198 353 353 353 F F F 5 100-119 193 193 193 341 341 341 F F F 6 120-139 198 198 198 351 351 351 F F F 7 140-159 198 198 198 353 353 353 F F F 8 160-179 199 199 199 354 354 354 F F F 9 180-199 198 198 198 350 350 350 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 196 196 196 347 347 347 F F F 12 240-259 195 195 195 346 346 346 F F F 13 260-279 197 197 197 348 348 348 F F F 14 280-299 194 194 194 345 345 345 F F F 15 300-319 194 194 194 345 345 345 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 198 198 198 352 352 352 F F F 19 380-399 198 198 198 353 353 353 F F F 20 400-419 198 198 198 351 351 351 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 196 196 196 347 347 347 F F F 23 460-479 192 192 192 340 340 340 F F F 24 480-499 192 192 192 340 340 340 F F F 25 500-519 196 196 196 347 347 347 F F F 26 520-539 197 197 197 348 348 348 F F F 27 540-559 199 199 199 354 354 354 F F F 28 560-579 198 198 198 351 351 351 F F F 29 580-599 198 198 198 353 353 353 F F F Max gen: 199, Max flush: 354 7:19 Test duration Test completed real 7:18.749375019 user 54.322076209 sys 17.386814844 trap 0.074923397 tflt 0.038008237 dflt 0.074374129 kflt 0.006726883 lock 11:47:43.053234933 slp 35:43.011936899 lat 5.195808382 stop 0.057905433