Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.3168094 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.048863508 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.050013136 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.745329687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.819768552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.821190965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.821290698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.821317013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.8213355Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188} {"msg":"Crucible 993e6646-c326-42ae-b2e0-100ae50fe1ab has session id: a7fea1f3-8f68-436d-a41e-688da9291459","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.822211167Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Upstairs opts: Upstairs UUID: 993e6646-c326-42ae-b2e0-100ae50fe1ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.822360381Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Crucible stats registered with UUID: 993e6646-c326-42ae-b2e0-100ae50fe1ab","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.822389956Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854417193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188} {"msg":"993e6646-c326-42ae-b2e0-100ae50fe1ab active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854477732Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854509197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.85460554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854646023Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854682137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854712372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854741307Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854769692Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854799356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854828031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_connection connected from Ok(127.0.0.1:44116)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854859736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_connection connected from Ok(127.0.0.1:59533)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.85489283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_connection connected from Ok(127.0.0.1:38931)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.854923675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.856697726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.856727481Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.856742418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.856756086Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.856777302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.856808587Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.858164191Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.858206264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.858232969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859476463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859511716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859533743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859553429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859575955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859596802Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859615838Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859637445Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859693905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859724099Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859740697Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859758924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859772841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859786909Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859800406Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859813814Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859827761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859840879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859853957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.859866915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860038145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"downstairs","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860061701Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860085037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860110682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860132988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"client":"2","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"Wait for a query_work_queue command to finish before sending IO msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860193538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860227782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"993e6646-c326-42ae-b2e0-100ae50fe1ab is now active with session: a7fea1f3-8f68-436d-a41e-688da9291459","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860243169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860255347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:09:56.860267535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 5 Acked 1004 Read 88 Done Done Done false 6 Acked 1005 Read 256 Done Done Done false 7 Acked 1006 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 7 6 5 4 3 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 8 7 6 5 4 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.099793501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188} {"msg":"client_stop_rx closed unexpectedly: RecvError(())","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.099921719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.099958263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.099988718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.100020222Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.100052306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.100080622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.100114565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.1001474Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"a7fea1f3-8f68-436d-a41e-688da9291459"} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.135594821Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.180680016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.181651676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.181698308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.181713026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.181727883Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190} {"msg":"Crucible 65e3969c-6563-47e7-b5c6-22c54ec89733 has session id: 3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.182238444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Upstairs opts: Upstairs UUID: 65e3969c-6563-47e7-b5c6-22c54ec89733, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.182294955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Crucible stats registered with UUID: 65e3969c-6563-47e7-b5c6-22c54ec89733","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.18232057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.213887497Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190} {"msg":"65e3969c-6563-47e7-b5c6-22c54ec89733 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.213950126Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.213991359Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.21403994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214076204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214106719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214210361Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214252413Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214289097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214321841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214348706Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44019)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214377661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46650)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214408636Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64526)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.214440761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.215202668Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.215234543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.21525031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.215270276Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.215431908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.215487368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.215973974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.21611365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216140215Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216589477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216616422Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216630669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216669803Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.21668717Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216700727Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216765416Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216784843Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216809938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216824736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216842523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.2168563Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216870188Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216884195Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216897513Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216910911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216924708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216938056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216951104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216963962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.216976689Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217003775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217024241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"0","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217044877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"1","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217067153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"client":"2","":"downstairs","session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217083151Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217101038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"65e3969c-6563-47e7-b5c6-22c54ec89733 is now active with session: 3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217123554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217150249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190,"session_id":"3a9c82e0-1192-490e-8c9e-c3dcdcb25c3c"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:01.217171115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 118 len 4096 data: 2 02/30 Write block 535 len 4096 data: 2 03/30 Read block 267 len 40960 04/30 Read block 429 len 28672 05/30 Read block 486 len 12288 06/30 Write block 233 len 16384 data: 2 2 2 2 07/30 Read block 461 len 24576 08/30 Read block 305 len 24576 09/30 Write block 70 len 32768 data: 2 2 2 2 2 2 2 2 10/30 Flush 11/30 Write block 344 len 4096 data: 2 12/30 Flush 13/30 Flush 14/30 Write block 341 len 8192 data: 2 2 15/30 Read block 433 len 28672 16/30 Read block 7 len 16384 17/30 Read block 329 len 28672 18/30 Read block 89 len 40960 19/30 Write block 44 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Read block 418 len 20480 21/30 Read block 88 len 36864 22/30 Write block 202 len 24576 data: 2 2 2 2 2 2 23/30 Write block 121 len 16384 data: 2 2 2 2 24/30 Write block 462 len 16384 data: 2 2 2 2 25/30 Write block 4 len 32768 data: 2 2 2 2 2 2 2 2 26/30 Read block 168 len 20480 27/30 Write block 41 len 12288 data: 2 2 2 28/30 Read block 7 len 20480 29/30 Read block 507 len 8192 30/30 Read block 496 len 12288 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 2 Done Done Done false 15 Acked 1014 Read 7 Done Done Done false 16 Acked 1015 Read 4 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Read 9 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:01.246663893Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 2 2 2 2 2 2 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 3 3 3 T T T 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 T T T 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 4 4 4 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 4 4 4 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 2 2 2 2 2 2 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 3 3 3 4 4 4 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.570174136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:15.613554717Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.614206823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.646115191Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.647031541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.647071215Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.647085592Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.647099959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198} {"msg":"Crucible db1daacc-d3cc-403b-a507-7ca5eeb04888 has session id: 11f44dd0-d469-4e6b-aa26-3066a807446e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.647569598Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Upstairs opts: Upstairs UUID: db1daacc-d3cc-403b-a507-7ca5eeb04888, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.647608241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Crucible stats registered with UUID: db1daacc-d3cc-403b-a507-7ca5eeb04888","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.647624378Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678664506Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198} {"msg":"db1daacc-d3cc-403b-a507-7ca5eeb04888 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678726956Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678746402Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678785945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678820639Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678854964Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678881139Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678908154Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678939189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678962545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.678980722Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44309)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.679075555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61683)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.679114318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49489)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.679144323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.68008366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.680111655Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.680126272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.68013972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.680157347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.680189261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.680773859Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.68094375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.680979334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681355658Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681385593Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.68140138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681589178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681618252Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.68163354Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681652546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681666444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[0]R flush_numbers[0..12]: [4, 1, 4, 2, 1, 2, 4, 1, 1, 1, 4, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681733783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[0]R generation[0..12]: [3, 1, 3, 2, 1, 2, 3, 1, 1, 1, 3, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681754379Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681768756Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[1]R flush_numbers[0..12]: [4, 1, 4, 2, 1, 2, 4, 1, 1, 1, 4, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681786173Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[1]R generation[0..12]: [3, 1, 3, 2, 1, 2, 3, 1, 1, 1, 3, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681800221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681814378Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[2]R flush_numbers[0..12]: [4, 1, 4, 2, 1, 2, 4, 1, 1, 1, 4, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681827926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[2]R generation[0..12]: [3, 1, 3, 2, 1, 2, 3, 1, 1, 1, 3, 2]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681841304Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681855361Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681868509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681881777Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681894615Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681907422Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"":"downstairs","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.68192051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Transition from WaitQuorum to ActiveWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681942686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"0","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681956104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"1","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681969122Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"client":"2","":"downstairs","session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681982199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.681996067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"db1daacc-d3cc-403b-a507-7ca5eeb04888 is now active with session: 11f44dd0-d469-4e6b-aa26-3066a807446e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.682009175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.682021142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198,"session_id":"11f44dd0-d469-4e6b-aa26-3066a807446e"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:15.68203335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 16384 data: 3 3 3 3 02/30 Write block 147 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Write block 276 len 4096 data: 2 04/30 Read block 362 len 4096 05/30 Write block 54 len 4096 data: 2 06/30 Flush 07/30 Write block 529 len 32768 data: 2 2 2 2 2 2 3 2 08/30 Read block 502 len 20480 09/30 Write block 530 len 8192 data: 3 3 10/30 Read block 302 len 32768 11/30 Read block 201 len 40960 12/30 Read block 10 len 12288 13/30 Read block 370 len 24576 14/30 Write block 178 len 4096 data: 2 15/30 Write block 139 len 8192 data: 2 2 16/30 Flush 17/30 Write block 228 len 4096 data: 2 18/30 Read block 280 len 40960 19/30 Write block 446 len 36864 data: 2 2 2 2 2 2 2 2 2 20/30 Flush 21/30 Write block 13 len 4096 data: 2 22/30 Write block 575 len 36864 data: 2 2 2 2 2 2 2 2 2 23/30 Write block 224 len 24576 data: 2 2 2 2 3 2 24/30 Write block 166 len 12288 data: 2 2 2 25/30 Write block 54 len 16384 data: 3 2 2 2 26/30 Read block 476 len 28672 27/30 Write block 106 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Write block 121 len 40960 data: 3 3 3 3 2 2 2 2 2 2 29/30 Read block 433 len 36864 30/30 Read block 489 len 16384 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:15.705000594Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 5 5 5 T T T 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 T T T 6 120-139 4 4 4 6 6 6 T T T 7 140-159 4 4 4 6 6 6 F F F 8 160-179 4 4 4 6 6 6 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 4 4 4 7 7 7 T T T 12 240-259 1 1 1 1 1 1 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 3 3 3 4 4 4 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 4 6 6 6 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 T T T 29 580-599 1 1 1 1 1 1 T T T Max gen: 4, Max flush: 7 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 4 4 4 6 6 6 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 3 3 3 4 4 4 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 4 6 6 6 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 5 5 5 8 8 8 F F F 29 580-599 5 5 5 8 8 8 F F F Max gen: 5, Max flush: 8 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.095934663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.141424887Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.142320271Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.142361194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.142376281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.142390719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206} {"msg":"Crucible d3b1efe4-b1b3-4905-bb6a-c062d8430c36 has session id: fe92f20d-712a-4bdb-b16a-1713f6f9bb58","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.142831632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Upstairs opts: Upstairs UUID: d3b1efe4-b1b3-4905-bb6a-c062d8430c36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.142876914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Crucible stats registered with UUID: d3b1efe4-b1b3-4905-bb6a-c062d8430c36","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.14289969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174286958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206} {"msg":"d3b1efe4-b1b3-4905-bb6a-c062d8430c36 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174356566Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.17439308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174441331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174475795Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174605073Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174624509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174639467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174666622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174698797Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174731251Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:34002)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174761735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:34014)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174795969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:52944)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.174835093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.175587112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.175630544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.175796855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.175836239Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.175866283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.175895318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.176258635Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.1764045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.176697789Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.176868689Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.176896534Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.176911571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.176970851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.176989988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177003565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177232136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177255032Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[0]R flush_numbers[0..12]: [8, 1, 8, 2, 1, 8, 8, 6, 8, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177271279Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[0]R generation[0..12]: [5, 1, 5, 2, 1, 5, 5, 4, 5, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177285007Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177299074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[1]R flush_numbers[0..12]: [8, 1, 8, 2, 1, 8, 8, 6, 8, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177312602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[1]R generation[0..12]: [5, 1, 5, 2, 1, 5, 5, 4, 5, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177325809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177341057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[2]R flush_numbers[0..12]: [8, 1, 8, 2, 1, 8, 8, 6, 8, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177354564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[2]R generation[0..12]: [5, 1, 5, 2, 1, 5, 5, 4, 5, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177371012Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177385049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177397957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177410954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177423822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.17743671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"":"downstairs","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177449698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177461686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","Wait for a query_work_queue command to finish before sending IO pid":1206,"client":"0","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177486491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"1","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177499379Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"client":"2","":"downstairs","session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177512527Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177524715Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"d3b1efe4-b1b3-4905-bb6a-c062d8430c36 is now active with session: fe92f20d-712a-4bdb-b16a-1713f6f9bb58","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177537363Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.17754919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206,"session_id":"fe92f20d-712a-4bdb-b16a-1713f6f9bb58"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:20.177562178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 12 len 32768 data: 2 3 2 2 2 2 2 2 02/30 Write block 496 len 16384 data: 2 2 2 2 03/30 Write block 31 len 4096 data: 2 04/30 Flush 05/30 Write block 471 len 8192 data: 2 2 06/30 Write block 168 len 24576 data: 3 2 2 2 2 2 07/30 Read block 267 len 4096 08/30 Write block 263 len 20480 data: 2 2 2 2 2 09/30 Read block 136 len 24576 10/30 Read block 321 len 12288 11/30 Write block 591 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Read block 286 len 36864 13/30 Write block 160 len 40960 data: 2 2 2 2 2 2 3 3 4 3 14/30 Read block 490 len 32768 15/30 Read block 554 len 28672 16/30 Read block 70 len 32768 17/30 Write block 12 len 32768 data: 3 4 3 3 3 3 3 3 18/30 Read block 116 len 12288 19/30 Write block 353 len 16384 data: 2 2 2 2 20/30 Write block 572 len 24576 data: 2 2 2 3 3 3 21/30 Write block 23 len 28672 data: 2 2 2 2 2 2 2 22/30 Read block 158 len 20480 23/30 Read block 443 len 40960 24/30 Write block 578 len 24576 data: 3 3 3 3 3 3 25/30 Flush 26/30 Write block 400 len 36864 data: 2 2 2 2 2 2 2 2 2 27/30 Read block 121 len 36864 28/30 Write block 446 len 12288 data: 3 3 3 29/30 Write block 410 len 32768 data: 2 2 2 2 2 2 2 2 30/30 Write block 449 len 20480 data: 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 3 Done Sent Done false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 2 7 Done 3 2 3 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:20.200636163Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 6 6 6 10 10 10 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 4 4 4 6 6 6 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 10 10 10 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 4 7 7 7 T T T 23 460-479 6 6 6 10 10 10 F F F 24 480-499 6 6 6 9 9 9 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 4 6 6 6 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 6, Max flush: 10 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 6 6 6 10 10 10 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 4 4 4 6 6 6 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 10 10 10 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 7 7 7 11 11 11 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 6 6 6 10 10 10 F F F 24 480-499 6 6 6 9 9 9 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 4 6 6 6 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 7, Max flush: 11 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.479570792Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.524789923Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.525693876Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.52572695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.525741168Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.525755615Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214} {"msg":"Crucible 373f7c94-09c9-416e-84ad-a30a07c5cd7a has session id: 2dde0e8d-1044-4158-ac66-d7878956ba0f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.526277284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Upstairs opts: Upstairs UUID: 373f7c94-09c9-416e-84ad-a30a07c5cd7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.526339764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Crucible stats registered with UUID: 373f7c94-09c9-416e-84ad-a30a07c5cd7a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.526369599Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557285508Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214} {"msg":"373f7c94-09c9-416e-84ad-a30a07c5cd7a active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.55733573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557351757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557387581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557416486Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557535755Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557591475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557634198Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557671331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557708055Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557746608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54254)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557785501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61976)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.557826594Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55876)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.55791125Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.558486739Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.558516954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.558531891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.558545429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.558693993Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.558780548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.559275762Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.559304837Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.559473238Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.55992118Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.559947995Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.559962543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560025462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560046288Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560065625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560084881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560098759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[0]R flush_numbers[0..12]: [10, 10, 8, 2, 1, 8, 8, 6, 10, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560165337Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[0]R generation[0..12]: [6, 6, 5, 2, 1, 5, 5, 4, 6, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560185974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560200411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[1]R flush_numbers[0..12]: [10, 10, 8, 2, 1, 8, 8, 6, 10, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560214149Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[1]R generation[0..12]: [6, 6, 5, 2, 1, 5, 5, 4, 6, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560228007Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560242204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[2]R flush_numbers[0..12]: [10, 10, 8, 2, 1, 8, 8, 6, 10, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560255782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[2]R generation[0..12]: [6, 6, 5, 2, 1, 5, 5, 4, 6, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560269409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560283237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560296115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560309122Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.56032199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560338097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"":"downstairs","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-09-27T17:10:24.560362363Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560374551Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"0","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560387709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"1","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560400626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"client":"2","":"downstairs","session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560413704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560425902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"373f7c94-09c9-416e-84ad-a30a07c5cd7a is now active with session: 2dde0e8d-1044-4158-ac66-d7878956ba0f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.56043915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560451208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214,"session_id":"2dde0e8d-1044-4158-ac66-d7878956ba0f"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:24.560463455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 412 len 36864 data: 3 3 3 3 3 3 2 2 2 02/30 Read block 302 len 16384 03/30 Read block 507 len 32768 04/30 Read block 427 len 24576 05/30 Write block 359 len 40960 data: 2 2 2 2 2 2 2 2 2 2 06/30 Write block 38 len 24576 data: 2 2 2 3 3 3 07/30 Write block 593 len 16384 data: 3 3 3 3 08/30 Read block 430 len 4096 09/30 Flush 10/30 Write block 550 len 16384 data: 2 2 2 2 11/30 Read block 31 len 36864 12/30 Flush 13/30 Write block 549 len 20480 data: 2 3 3 3 3 14/30 Write block 529 len 32768 data: 3 4 4 3 3 3 4 3 15/30 Read block 345 len 20480 16/30 Read block 515 len 12288 17/30 Write block 548 len 40960 data: 2 3 4 4 4 4 2 2 2 2 18/30 Write block 515 len 12288 data: 2 2 2 19/30 Read block 272 len 40960 20/30 Write block 256 len 20480 data: 2 2 2 2 2 21/30 Write block 522 len 16384 data: 2 2 2 2 22/30 Write block 494 len 28672 data: 2 2 3 3 3 3 2 23/30 Write block 406 len 16384 data: 3 3 3 2 24/30 Read block 320 len 24576 25/30 Flush 26/30 Write block 30 len 40960 data: 2 3 2 2 2 2 2 2 3 3 27/30 Read block 135 len 8192 28/30 Write block 79 len 4096 data: 2 29/30 Read block 16 len 24576 30/30 Write block 137 len 24576 data: 2 2 3 3 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:24.586320496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 8 8 8 12 12 12 T T T 2 040-059 8 8 8 12 12 12 F F F 3 060-079 2 2 2 2 2 2 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 T T T 7 140-159 4 4 4 6 6 6 T T T 8 160-179 6 6 6 10 10 10 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 8 8 8 14 14 14 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 8 8 8 12 12 12 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 8 8 8 14 14 14 F F F 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 6 6 6 10 10 10 F F F 24 480-499 8 8 8 14 14 14 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 8 8 8 14 14 14 F F F 27 540-559 8 8 8 14 14 14 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 8, Max flush: 14 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 9 9 9 15 15 15 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 9 9 9 15 15 15 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 9 9 9 15 15 15 F F F 7 140-159 9 9 9 15 15 15 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 8 8 8 14 14 14 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 8 8 8 12 12 12 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 8 8 8 14 14 14 F F F 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 6 6 6 10 10 10 F F F 24 480-499 8 8 8 14 14 14 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 8 8 8 14 14 14 F F F 27 540-559 8 8 8 14 14 14 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.895536625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.940400038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.941358511Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.941412931Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.941431318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.941446386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222} {"msg":"Crucible fc487123-1ad6-465a-a6e0-8117ad60d1e1 has session id: 45beff56-745b-40d5-85d6-81935ccd163b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.941953937Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Upstairs opts: Upstairs UUID: fc487123-1ad6-465a-a6e0-8117ad60d1e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.941989591Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Crucible stats registered with UUID: fc487123-1ad6-465a-a6e0-8117ad60d1e1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.942004458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973262799Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222} {"msg":"fc487123-1ad6-465a-a6e0-8117ad60d1e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973332247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973368351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973422871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973459355Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973496138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973530722Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973568196Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.97360074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973632985Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973664149Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45388)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973694174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36312)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973778709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41046)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.973960448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.974455991Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.974485696Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.974501283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.974515361Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.97475064Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.974787453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975222498Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975258851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975581765Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975892761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975929435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975954421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975975407Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.975998293Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976022479Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976207936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.97624495Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[0]R flush_numbers[0..12]: [10, 15, 12, 15, 1, 8, 15, 15, 10, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976269806Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[0]R generation[0..12]: [6, 9, 8, 9, 1, 5, 9, 9, 6, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976295441Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976325626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[1]R flush_numbers[0..12]: [10, 15, 12, 15, 1, 8, 15, 15, 10, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976348722Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[1]R generation[0..12]: [6, 9, 8, 9, 1, 5, 9, 9, 6, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976371358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976396204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[2]R flush_numbers[0..12]: [10, 15, 12, 15, 1, 8, 15, 15, 10, 1, 4, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976420869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[2]R generation[0..12]: [6, 9, 8, 9, 1, 5, 9, 9, 6, 1, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976446735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.9764714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976495576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976522771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976562105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.97659116Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"":"downstairs","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976617805Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976642081Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"0","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976668826Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"1","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976696321Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"client":"2","":"downstairs","session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976727056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976752162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"fc487123-1ad6-465a-a6e0-8117ad60d1e1 is now active with session: 45beff56-745b-40d5-85d6-81935ccd163b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976769239Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976782366Z","hostname":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222,"session_id":"45beff56-745b-40d5-85d6-81935ccd163b"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:28.976799833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 507 len 20480 data: 2 2 2 2 2 02/30 Flush 03/30 Write block 15 len 28672 data: 4 4 4 4 4 2 2 04/30 Write block 530 len 16384 data: 5 5 4 4 05/30 Read block 153 len 4096 06/30 Flush 07/30 Write block 462 len 20480 data: 3 3 3 3 2 08/30 Write block 324 len 20480 data: 2 2 2 2 2 09/30 Read block 100 len 36864 10/30 Write block 94 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Write block 346 len 32768 data: 2 2 2 2 2 2 2 3 12/30 Read block 202 len 8192 13/30 Read block 31 len 36864 14/30 Read block 109 len 16384 15/30 Read block 337 len 36864 16/30 Write block 84 len 4096 data: 2 17/30 Write block 226 len 20480 data: 3 3 4 3 2 18/30 Read block 279 len 16384 19/30 Read block 293 len 16384 20/30 Read block 154 len 8192 21/30 Read block 440 len 24576 22/30 Read block 134 len 24576 23/30 Flush 24/30 Write block 30 len 20480 data: 3 4 3 3 3 25/30 Read block 311 len 40960 26/30 Read block 88 len 12288 27/30 Read block 350 len 36864 28/30 Write block 415 len 12288 data: 4 4 4 29/30 Write block 194 len 24576 data: 2 2 2 2 2 2 30/30 Read block 399 len 8192 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 6 Done Done Done false 30 Acked 1029 Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:29.002465537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 10 10 10 17 17 17 T T T 2 040-059 8 8 8 12 12 12 F F F 3 060-079 9 9 9 15 15 15 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 18 18 18 F F F 6 120-139 9 9 9 15 15 15 F F F 7 140-159 9 9 9 15 15 15 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 1 1 1 1 1 1 T T T 10 200-219 3 3 3 4 4 4 F F F 11 220-239 10 10 10 18 18 18 F F F 12 240-259 8 8 8 14 14 14 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 10 10 10 18 18 18 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 8 8 8 14 14 14 T T T 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 10 10 10 18 18 18 F F F 24 480-499 8 8 8 14 14 14 F F F 25 500-519 10 10 10 16 16 16 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 8 8 8 14 14 14 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 10, Max flush: 18 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 11 11 11 19 19 19 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 9 9 9 15 15 15 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 18 18 18 F F F 6 120-139 9 9 9 15 15 15 F F F 7 140-159 9 9 9 15 15 15 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 11 11 11 19 19 19 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 10 10 10 18 18 18 F F F 12 240-259 8 8 8 14 14 14 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 10 10 10 18 18 18 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 11 11 11 19 19 19 F F F 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 10 10 10 18 18 18 F F F 24 480-499 8 8 8 14 14 14 F F F 25 500-519 10 10 10 16 16 16 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 8 8 8 14 14 14 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 11, Max flush: 19 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.287915753Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.333114768Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.333941474Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.333978227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.333997464Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.334012812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230} {"msg":"Crucible 1047a6de-2fde-40a5-86a8-64f49357887b has session id: 7826a322-a25b-421b-8e78-db39b5fd94a4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.334404433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Upstairs opts: Upstairs UUID: 1047a6de-2fde-40a5-86a8-64f49357887b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.334447726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Crucible stats registered with UUID: 1047a6de-2fde-40a5-86a8-64f49357887b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.334475041Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365527707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230} {"msg":"1047a6de-2fde-40a5-86a8-64f49357887b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365572829Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365602034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365648886Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.36568442Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365721554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365755917Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365791821Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365828555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365863889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.365898893Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63663)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.366148489Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:57683)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.366196021Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47285)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.366229845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.366710801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.366751184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.366781879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.366811654Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.367010369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.367040164Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.36758075Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.367624772Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.367653118Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368126205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.36815375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368209861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368235836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368251263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368264851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368369423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368389539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[0]R flush_numbers[0..12]: [17, 19, 12, 15, 18, 18, 15, 15, 10, 19, 4, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368412845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[0]R generation[0..12]: [10, 11, 8, 9, 10, 10, 9, 9, 6, 11, 3, 10]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368427683Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.36844204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[1]R flush_numbers[0..12]: [17, 19, 12, 15, 18, 18, 15, 15, 10, 19, 4, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368455748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[1]R generation[0..12]: [10, 11, 8, 9, 10, 10, 9, 9, 6, 11, 3, 10]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368469225Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368483353Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[2]R flush_numbers[0..12]: [17, 19, 12, 15, 18, 18, 15, 15, 10, 19, 4, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.36849706Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[2]R generation[0..12]: [10, 11, 8, 9, 10, 10, 9, 9, 6, 11, 3, 10]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368513898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368528065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368540833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368553941Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368566779Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368580206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"":"downstairs","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368593334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368605272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"0","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368618499Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"1","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368645865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"client":"2","":"downstairs","session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368659082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.36867133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"1047a6de-2fde-40a5-86a8-64f49357887b is now active with session: 7826a322-a25b-421b-8e78-db39b5fd94a4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368684328Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368696456Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230,"session_id":"7826a322-a25b-421b-8e78-db39b5fd94a4"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:33.368708784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 535 len 32768 data: 5 4 2 2 2 2 2 2 02/30 Write block 516 len 36864 data: 3 3 2 2 2 2 3 3 3 03/30 Flush 04/30 Write block 524 len 8192 data: 4 3 05/30 Flush 06/30 Write block 472 len 28672 data: 3 2 2 2 2 2 2 07/30 Flush 08/30 Write block 266 len 32768 data: 3 3 2 2 2 2 2 2 09/30 Write block 202 len 12288 data: 3 3 3 10/30 Write block 70 len 28672 data: 3 3 3 3 3 3 3 11/30 Write block 537 len 28672 data: 3 3 3 3 3 3 2 12/30 Write block 370 len 24576 data: 2 2 2 2 2 2 13/30 Read block 285 len 40960 14/30 Flush 15/30 Write block 468 len 4096 data: 2 16/30 Read block 416 len 40960 17/30 Read block 37 len 36864 18/30 Read block 477 len 36864 19/30 Read block 356 len 4096 20/30 Write block 561 len 4096 data: 2 21/30 Write block 328 len 36864 data: 3 2 2 2 2 2 2 2 2 22/30 Read block 468 len 36864 23/30 Read block 199 len 16384 24/30 Flush 25/30 Write block 310 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Write block 134 len 4096 data: 2 27/30 Write block 362 len 24576 data: 3 3 3 3 3 3 28/30 Read block 417 len 28672 29/30 Read block 126 len 36864 30/30 Write block 180 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:33.397619873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 11 11 11 19 19 19 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 12 12 12 23 23 23 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 18 18 18 F F F 6 120-139 9 9 9 15 15 15 T T T 7 140-159 9 9 9 15 15 15 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 11 11 11 19 19 19 T T T 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 18 18 18 F F F 12 240-259 8 8 8 14 14 14 F F F 13 260-279 12 12 12 23 23 23 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 12 12 12 24 24 24 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 12 12 12 23 23 23 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 11 11 11 19 19 19 F F F 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 12 12 12 24 24 24 F F F 24 480-499 8 8 8 14 14 14 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 12 12 12 23 23 23 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 12 12 12 24 24 24 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 12, Max flush: 24 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 11 11 11 19 19 19 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 12 12 12 23 23 23 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 18 18 18 F F F 6 120-139 13 13 13 25 25 25 F F F 7 140-159 9 9 9 15 15 15 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 13 13 13 25 25 25 F F F 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 18 18 18 F F F 12 240-259 8 8 8 14 14 14 F F F 13 260-279 12 12 12 23 23 23 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 13 13 13 25 25 25 F F F 16 320-339 12 12 12 24 24 24 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 11 11 11 19 19 19 F F F 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 12 12 12 24 24 24 F F F 24 480-499 8 8 8 14 14 14 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 12 12 12 23 23 23 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 12 12 12 24 24 24 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 13, Max flush: 25 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.692650002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.737239142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.738052391Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.738122908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.738148164Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.73817274Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238} {"msg":"Crucible cbea0017-a48f-45ff-9fc0-29602469218f has session id: ec79a94e-b0c5-445d-aee0-582ab98d0ce3","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.73863247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Upstairs opts: Upstairs UUID: cbea0017-a48f-45ff-9fc0-29602469218f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.738706547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Crucible stats registered with UUID: cbea0017-a48f-45ff-9fc0-29602469218f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.738727593Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770144626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238} {"msg":"cbea0017-a48f-45ff-9fc0-29602469218f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770212194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770246248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.77029555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770334633Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770367477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770398462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770425377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770453042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770483037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770513702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39301)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770546156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60067)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770575351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49932)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.770612975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.77126959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.771297685Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.771313193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.77132744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.771756005Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.771795979Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772030808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772165124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772500796Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772629114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772665097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772680565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772783227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772802203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.772819891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773124657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773143744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[0]R flush_numbers[0..12]: [17, 19, 12, 23, 18, 18, 25, 15, 10, 25, 23, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773171729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[0]R generation[0..12]: [10, 11, 8, 12, 10, 10, 13, 9, 6, 13, 12, 10]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773186337Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773200384Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[1]R flush_numbers[0..12]: [17, 19, 12, 23, 18, 18, 25, 15, 10, 25, 23, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773214032Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[1]R generation[0..12]: [10, 11, 8, 12, 10, 10, 13, 9, 6, 13, 12, 10]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773227449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773241287Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[2]R flush_numbers[0..12]: [17, 19, 12, 23, 18, 18, 25, 15, 10, 25, 23, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773254775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[2]R generation[0..12]: [10, 11, 8, 12, 10, 10, 13, 9, 6, 13, 12, 10]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773268193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.77328191Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773294668Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773307726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773323633Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.77333671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"":"downstairs","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773349798Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773361796Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"0","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773388141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"1","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773401289Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"client":"2","":"downstairs","session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773414307Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773426355Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"cbea0017-a48f-45ff-9fc0-29602469218f is now active with session: ec79a94e-b0c5-445d-aee0-582ab98d0ce3","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773439013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773450761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238,"session_id":"ec79a94e-b0c5-445d-aee0-582ab98d0ce3"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:37.773462728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 254 len 28672 data: 2 2 3 3 3 3 3 03/30 Read block 385 len 40960 04/30 Flush 05/30 Write block 260 len 40960 data: 4 2 2 3 3 3 4 4 3 3 06/30 Write block 375 len 20480 data: 3 2 2 2 2 07/30 Read block 325 len 16384 08/30 Write block 185 len 24576 data: 3 3 2 2 2 2 09/30 Flush 10/30 Write block 114 len 24576 data: 2 2 2 2 3 2 11/30 Read block 412 len 4096 12/30 Read block 437 len 24576 13/30 Read block 446 len 40960 14/30 Read block 193 len 8192 15/30 Read block 157 len 20480 16/30 Write block 565 len 20480 data: 2 2 2 2 2 17/30 Read block 144 len 16384 18/30 Write block 497 len 12288 data: 4 4 4 19/30 Read block 451 len 36864 20/30 Write block 338 len 12288 data: 2 2 2 21/30 Read block 505 len 40960 22/30 Write block 286 len 16384 data: 2 2 2 2 23/30 Read block 322 len 24576 24/30 Write block 82 len 24576 data: 2 2 3 2 2 2 25/30 Write block 213 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Read block 95 len 32768 27/30 Write block 567 len 8192 data: 3 3 28/30 Write block 136 len 28672 data: 2 3 3 4 4 3 3 29/30 Read block 58 len 32768 30/30 Write block 529 len 16384 data: 4 6 6 5 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 10 Acked 1009 Write 6 Done Done Done false 11 Acked 1010 Read 1 Done Done Done false 12 Acked 1011 Read 6 Done Done Done false 13 Acked 1012 Read 10 Done Done Done false 14 Acked 1013 Read 2 Done Done Done false 15 Acked 1014 Read 5 Done Done Done false 16 Acked 1015 Write 5 Done Done Done false 17 Acked 1016 Read 4 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Read 9 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:37.797218214Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 11 11 11 19 19 19 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 12 12 12 23 23 23 F F F 4 080-099 10 10 10 18 18 18 T T T 5 100-119 10 10 10 18 18 18 T T T 6 120-139 13 13 13 25 25 25 T T T 7 140-159 9 9 9 15 15 15 T T T 8 160-179 6 6 6 10 10 10 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 12 12 12 23 23 23 T T T 11 220-239 10 10 10 18 18 18 T T T 12 240-259 14 14 14 27 27 27 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 13 13 13 25 25 25 F F F 16 320-339 12 12 12 24 24 24 T T T 17 340-359 10 10 10 18 18 18 T T T 18 360-379 14 14 14 28 28 28 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 11 11 11 19 19 19 F F F 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 12 12 12 24 24 24 F F F 24 480-499 8 8 8 14 14 14 T T T 25 500-519 12 12 12 20 20 20 F F F 26 520-539 12 12 12 23 23 23 T T T 27 540-559 12 12 12 23 23 23 F F F 28 560-579 12 12 12 24 24 24 T T T 29 580-599 8 8 8 12 12 12 F F F Max gen: 14, Max flush: 28 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 11 11 11 19 19 19 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 12 12 12 23 23 23 F F F 4 080-099 15 15 15 29 29 29 F F F 5 100-119 15 15 15 29 29 29 F F F 6 120-139 15 15 15 29 29 29 F F F 7 140-159 15 15 15 29 29 29 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 15 15 15 29 29 29 F F F 12 240-259 14 14 14 27 27 27 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 15 15 15 29 29 29 F F F 15 300-319 13 13 13 25 25 25 F F F 16 320-339 15 15 15 29 29 29 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 14 14 14 28 28 28 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 11 11 11 19 19 19 F F F 21 420-439 8 8 8 12 12 12 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 12 12 12 24 24 24 F F F 24 480-499 15 15 15 29 29 29 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 15 15 15 29 29 29 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 15 15 15 29 29 29 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 15, Max flush: 29 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.304756212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:42.348812995Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.34930338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.381191771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.382120269Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.382165461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.382182648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.382200265Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246} {"msg":"Crucible 59dcc5fc-594c-4866-a819-92f01fd149b9 has session id: 94630372-b1e3-4080-b392-2f25bee28770","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.382734532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Upstairs opts: Upstairs UUID: 59dcc5fc-594c-4866-a819-92f01fd149b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.382792012Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Crucible stats registered with UUID: 59dcc5fc-594c-4866-a819-92f01fd149b9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.382814418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.41372276Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246} {"msg":"59dcc5fc-594c-4866-a819-92f01fd149b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413772911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413792828Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413824962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413843219Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413860206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413876523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.41389342Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413910897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.413990893Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.414036725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_connection connected from Ok(127.0.0.1:39968)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.414077498Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_connection connected from Ok(127.0.0.1:53136)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.414109802Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_connection connected from Ok(127.0.0.1:44523)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.414145036Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415075234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415117037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415148152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415177876Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415205382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415230107Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415681499Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.415827543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416010682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416312189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416341534Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416357291Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416417861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416445656Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416464372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416558266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416579963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[0]R flush_numbers[0..12]: [17, 19, 12, 23, 29, 29, 29, 29, 10, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416601789Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[0]R generation[0..12]: [10, 11, 8, 12, 15, 15, 15, 15, 6, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416616986Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416631443Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[1]R flush_numbers[0..12]: [17, 19, 12, 23, 29, 29, 29, 29, 10, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416645501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[1]R generation[0..12]: [10, 11, 8, 12, 15, 15, 15, 15, 6, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416659129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416673546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[2]R flush_numbers[0..12]: [17, 19, 12, 23, 29, 29, 29, 29, 10, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416687194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[2]R generation[0..12]: [10, 11, 8, 12, 15, 15, 15, 15, 6, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416700571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416714319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416727097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416740455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416755862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416768879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"":"downstairs","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416782477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416794545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","Wait for a query_work_queue command to finish before sending IO pid":1246,"client":"0","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.41682567Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"1","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416851895Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"client":"2","":"downstairs","session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416877521Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416900796Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"59dcc5fc-594c-4866-a819-92f01fd149b9 is now active with session: 94630372-b1e3-4080-b392-2f25bee28770","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416928062Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416952538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246,"session_id":"94630372-b1e3-4080-b392-2f25bee28770"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:42.416973554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 508 len 36864 data: 3 3 3 3 2 2 2 3 4 02/30 Read block 3 len 4096 03/30 Read block 222 len 24576 04/30 Write block 17 len 32768 data: 5 5 5 3 3 2 3 3 05/30 Read block 344 len 24576 06/30 Read block 328 len 24576 07/30 Read block 430 len 4096 08/30 Write block 419 len 28672 data: 3 3 2 2 2 2 2 09/30 Read block 12 len 40960 10/30 Read block 159 len 8192 11/30 Write block 311 len 4096 data: 3 12/30 Write block 374 len 12288 data: 3 4 3 13/30 Write block 282 len 4096 data: 2 14/30 Read block 532 len 24576 15/30 Read block 363 len 12288 16/30 Read block 134 len 40960 17/30 Write block 118 len 36864 data: 4 3 2 4 4 4 4 3 3 18/30 Read block 414 len 40960 19/30 Read block 124 len 24576 20/30 Read block 307 len 40960 21/30 Read block 471 len 8192 22/30 Read block 284 len 8192 23/30 Read block 51 len 16384 24/30 Write block 358 len 28672 data: 2 3 3 3 4 4 4 25/30 Read block 532 len 12288 26/30 Read block 84 len 4096 27/30 Write block 125 len 24576 data: 4 4 3 3 3 3 28/30 Read block 138 len 28672 29/30 Write block 40 len 36864 data: 3 4 4 4 3 3 3 3 3 30/30 Write block 513 len 16384 data: 3 3 4 5 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 9 Done Done Done false 2 Acked 1001 Read 1 Done Done Done false 3 Acked 1002 Read 6 Done Done Done false 4 Acked 1003 Write 8 Done Done Done false 5 Acked 1004 Read 6 Done Done Done false 6 Acked 1005 Read 6 Done Done Done false 7 Acked 1006 Read 1 Done Done Done false 8 Acked 1007 Write 7 Done Done Done false 9 Acked 1008 Read 10 Done Done Done false 10 Acked 1009 Read 2 Done Done Done false 11 Acked 1010 Write 1 Done Done Done false 12 Acked 1011 Write 3 Done Done Done false 13 Acked 1012 Write 1 Done Done Done false 14 Acked 1013 Read 6 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 10 Done Done Done false 17 Acked 1016 Write 9 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:42.437617111Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 T T T 1 020-039 11 11 11 19 19 19 T T T 2 040-059 8 8 8 12 12 12 T T T 3 060-079 12 12 12 23 23 23 F F F 4 080-099 15 15 15 29 29 29 F F F 5 100-119 15 15 15 29 29 29 T T T 6 120-139 15 15 15 29 29 29 T T T 7 140-159 15 15 15 29 29 29 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 15 15 15 29 29 29 F F F 12 240-259 14 14 14 27 27 27 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 15 15 15 29 29 29 T T T 15 300-319 13 13 13 25 25 25 T T T 16 320-339 15 15 15 29 29 29 F F F 17 340-359 15 15 15 29 29 29 T T T 18 360-379 14 14 14 28 28 28 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 11 11 11 19 19 19 T T T 21 420-439 8 8 8 12 12 12 T T T 22 440-459 7 7 7 11 11 11 F F F 23 460-479 12 12 12 24 24 24 F F F 24 480-499 15 15 15 29 29 29 F F F 25 500-519 12 12 12 20 20 20 T T T 26 520-539 15 15 15 29 29 29 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 15 15 15 29 29 29 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 15, Max flush: 29 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 16 16 16 30 30 30 F F F 3 060-079 12 12 12 23 23 23 F F F 4 080-099 15 15 15 29 29 29 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 16 16 16 30 30 30 F F F 7 140-159 15 15 15 29 29 29 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 15 15 15 29 29 29 F F F 12 240-259 14 14 14 27 27 27 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 16 16 16 30 30 30 F F F 15 300-319 16 16 16 30 30 30 F F F 16 320-339 15 15 15 29 29 29 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 16 16 16 30 30 30 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 12 12 12 24 24 24 F F F 24 480-499 15 15 15 29 29 29 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 15 15 15 29 29 29 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 15 15 15 29 29 29 F F F 29 580-599 8 8 8 12 12 12 F F F Max gen: 16, Max flush: 30 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.853637365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.898394897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.899242309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.899275513Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.899289291Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.899303459Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254} {"msg":"Crucible 92fe9f6d-8d30-4957-a27b-ff4a73f40d4e has session id: 4a107907-3e1a-4a88-b68a-3a265c06b89b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.899833486Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Upstairs opts: Upstairs UUID: 92fe9f6d-8d30-4957-a27b-ff4a73f40d4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.89986728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Crucible stats registered with UUID: 92fe9f6d-8d30-4957-a27b-ff4a73f40d4e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.899884178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931162604Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254} {"msg":"92fe9f6d-8d30-4957-a27b-ff4a73f40d4e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931216475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931233972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931264237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931281684Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931298291Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931314718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931331515Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931347582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931364539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931381436Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57954)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.93153447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56813)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931592229Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41503)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.931858903Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.932518179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.932558052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.932605983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.932640407Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.932670372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.932699067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.93319808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933242612Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933374929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933826981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933867084Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933894759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933921944Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933952129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933974725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.933995231Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934018487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[0]R flush_numbers[0..12]: [30, 30, 30, 23, 29, 30, 30, 29, 10, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934043633Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[0]R generation[0..12]: [16, 16, 16, 12, 15, 16, 16, 15, 6, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934067719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934090245Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[1]R flush_numbers[0..12]: [30, 30, 30, 23, 29, 30, 30, 29, 10, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934113291Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[1]R generation[0..12]: [16, 16, 16, 12, 15, 16, 16, 15, 6, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934137897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934164832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[2]R flush_numbers[0..12]: [30, 30, 30, 23, 29, 30, 30, 29, 10, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934190807Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[2]R generation[0..12]: [16, 16, 16, 12, 15, 16, 16, 15, 6, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934213953Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934241728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Max found gen is 17","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-09-27T17:10:46.934280272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Generation requested: 18 >= found:17","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934311007Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934341121Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934368446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"":"downstairs","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934396052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934421777Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"0","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934449172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"1","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934480507Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"client":"2","":"downstairs","session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934508152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934532238Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"92fe9f6d-8d30-4957-a27b-ff4a73f40d4e is now active with session: 4a107907-3e1a-4a88-b68a-3a265c06b89b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934546666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934558753Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254,"session_id":"4a107907-3e1a-4a88-b68a-3a265c06b89b"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:46.934571021Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 285 len 20480 data: 2 3 3 3 3 02/30 Read block 367 len 40960 03/30 Read block 347 len 16384 04/30 Read block 524 len 8192 05/30 Read block 164 len 36864 06/30 Write block 96 len 36864 data: 3 3 3 3 3 3 3 3 2 07/30 Write block 51 len 20480 data: 3 3 3 4 3 08/30 Write block 371 len 40960 data: 3 3 3 4 5 4 3 3 3 2 09/30 Write block 585 len 40960 data: 2 2 2 2 2 2 3 3 4 4 10/30 Write block 70 len 28672 data: 4 4 4 4 4 4 4 11/30 Read block 21 len 36864 12/30 Write block 528 len 12288 data: 2 5 7 13/30 Read block 333 len 24576 14/30 Read block 313 len 4096 15/30 Read block 534 len 24576 16/30 Flush 17/30 Write block 233 len 32768 data: 3 3 3 3 2 2 2 2 18/30 Write block 83 len 16384 data: 3 4 3 3 19/30 Read block 245 len 40960 20/30 Read block 390 len 20480 21/30 Write block 474 len 16384 data: 3 3 3 3 22/30 Flush 23/30 Write block 304 len 28672 data: 2 2 2 2 2 2 3 24/30 Flush 25/30 Write block 421 len 8192 data: 3 3 26/30 Write block 583 len 20480 data: 4 2 3 3 3 27/30 Read block 112 len 36864 28/30 Read block 12 len 36864 29/30 Write block 497 len 24576 data: 5 5 5 3 2 2 30/30 Read block 294 len 28672 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 6 Done Done Sent false 30 Acked 1029 Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 5 6 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:46.965622368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 18 18 18 31 31 31 F F F 4 080-099 18 18 18 32 32 32 F F F 5 100-119 18 18 18 31 31 31 F F F 6 120-139 16 16 16 30 30 30 F F F 7 140-159 15 15 15 29 29 29 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 18 18 18 32 32 32 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 18 18 18 33 33 33 F F F 16 320-339 15 15 15 29 29 29 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 18 18 18 31 31 31 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 16 16 16 30 30 30 T T T 22 440-459 7 7 7 11 11 11 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 15 15 15 29 29 29 T T T 25 500-519 16 16 16 30 30 30 T T T 26 520-539 18 18 18 31 31 31 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 15 15 15 29 29 29 F F F 29 580-599 18 18 18 31 31 31 T T T Max gen: 18, Max flush: 33 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 18 18 18 31 31 31 F F F 4 080-099 18 18 18 32 32 32 F F F 5 100-119 18 18 18 31 31 31 F F F 6 120-139 16 16 16 30 30 30 F F F 7 140-159 15 15 15 29 29 29 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 18 18 18 32 32 32 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 18 18 18 33 33 33 F F F 16 320-339 15 15 15 29 29 29 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 18 18 18 31 31 31 F F F 20 400-419 16 16 16 30 30 30 F F F 21 420-439 19 19 19 34 34 34 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 19 19 19 34 34 34 F F F 26 520-539 18 18 18 31 31 31 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 15 15 15 29 29 29 F F F 29 580-599 19 19 19 34 34 34 F F F Max gen: 19, Max flush: 34 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.287270575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:51.330889694Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.331534502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.364312638Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.36527533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.365322532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.365340459Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.365358586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262} {"msg":"Crucible 87a5d16b-9d04-474e-9e15-a195315416dd has session id: 8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.365864418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Upstairs opts: Upstairs UUID: 87a5d16b-9d04-474e-9e15-a195315416dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.365899482Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Crucible stats registered with UUID: 87a5d16b-9d04-474e-9e15-a195315416dd","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.365915009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397060649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262} {"msg":"87a5d16b-9d04-474e-9e15-a195315416dd active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.39711017Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397126098Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397146724Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397160871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397174659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397188217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397201735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397215042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397232889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397257875Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58834)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397371525Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53800)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397413538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38045)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.397447462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.398321729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.398358053Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.39837362Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.398387328Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.398629906Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.398657261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399175001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399204926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399221143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399817229Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399847024Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399862532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399875239Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399888947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399902494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399914552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.39992763Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[0]R flush_numbers[0..12]: [30, 30, 31, 31, 32, 31, 30, 29, 10, 28, 29, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399944657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[0]R generation[0..12]: [16, 16, 18, 18, 18, 18, 16, 15, 6, 14, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399959195Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.399973692Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[1]R flush_numbers[0..12]: [30, 30, 31, 31, 32, 31, 30, 29, 10, 28, 29, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.39998735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[1]R generation[0..12]: [16, 16, 18, 18, 18, 18, 16, 15, 6, 14, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400031702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400049829Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[2]R flush_numbers[0..12]: [30, 30, 31, 31, 32, 31, 30, 29, 10, 28, 29, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400063676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[2]R generation[0..12]: [16, 16, 18, 18, 18, 18, 16, 15, 6, 14, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400077114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400090972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400103839Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400116857Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400129395Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"All extents match",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.40015557Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"":"downstairs","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400169048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400183476Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"0","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400197173Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"1","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400210351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"client":"2","":"downstairs","session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400223718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400235967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"87a5d16b-9d04-474e-9e15-a195315416dd is now active with session: 8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400248784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.400260592Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262,"session_id":"8d1540ee-aaf1-4c9b-b0d8-3aca5df3c66b"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:51.40027275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 4096 data: 2 02/30 Write block 248 len 40960 data: 2 2 2 2 2 2 3 3 4 4 03/30 Write block 410 len 36864 data: 3 3 4 4 4 5 5 5 3 04/30 Write block 181 len 28672 data: 3 3 3 3 4 4 3 05/30 Read block 457 len 24576 06/30 Read block 528 len 12288 07/30 Read block 428 len 32768 08/30 Read block 250 len 20480 09/30 Write block 508 len 8192 data: 4 4 10/30 Flush 11/30 Write block 250 len 8192 data: 3 3 12/30 Write block 39 len 4096 data: 4 13/30 Write block 102 len 20480 data: 4 4 3 2 3 14/30 Read block 443 len 16384 15/30 Read block 53 len 36864 16/30 Read block 368 len 36864 17/30 Write block 446 len 20480 data: 4 4 4 4 4 18/30 Write block 106 len 28672 data: 4 3 3 3 3 3 3 19/30 Write block 369 len 24576 data: 2 3 4 4 4 5 20/30 Write block 516 len 28672 data: 6 4 3 3 3 3 4 21/30 Flush 22/30 Write block 66 len 28672 data: 2 2 2 2 5 5 5 23/30 Read block 457 len 24576 24/30 Read block 463 len 36864 25/30 Read block 112 len 4096 26/30 Read block 441 len 24576 27/30 Read block 528 len 20480 28/30 Write block 540 len 4096 data: 4 29/30 Read block 201 len 16384 30/30 Read block 599 len 4096 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:51.422427575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 20 20 20 36 36 36 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 18 18 18 31 31 31 T T T 4 080-099 18 18 18 32 32 32 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 16 16 16 30 30 30 F F F 7 140-159 15 15 15 29 29 29 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 20 20 20 35 35 35 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 20 20 20 36 36 36 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 18 18 18 33 33 33 F F F 16 320-339 15 15 15 29 29 29 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 18 18 18 31 31 31 F F F 20 400-419 20 20 20 35 35 35 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 20 20 20 36 36 36 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 20 20 20 36 36 36 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 12 12 12 23 23 23 T T T 28 560-579 15 15 15 29 29 29 F F F 29 580-599 19 19 19 34 34 34 F F F Max gen: 20, Max flush: 36 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 20 20 20 36 36 36 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 21 21 21 37 37 37 F F F 4 080-099 18 18 18 32 32 32 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 16 16 16 30 30 30 F F F 7 140-159 15 15 15 29 29 29 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 20 20 20 35 35 35 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 20 20 20 36 36 36 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 18 18 18 33 33 33 F F F 16 320-339 15 15 15 29 29 29 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 18 18 18 31 31 31 F F F 20 400-419 20 20 20 35 35 35 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 20 20 20 36 36 36 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 20 20 20 36 36 36 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 15 15 15 29 29 29 F F F 29 580-599 19 19 19 34 34 34 F F F Max gen: 21, Max flush: 37 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.729637324Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:55.773502981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.774072992Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.806500108Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.807366977Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.807403971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.807417848Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.807432296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270} {"msg":"Crucible c41cd979-1eab-475b-a42b-b8cc8e74614a has session id: b399a6c2-2506-4c74-8c58-b36d8230c7f1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.807883927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Upstairs opts: Upstairs UUID: c41cd979-1eab-475b-a42b-b8cc8e74614a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.80792862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Crucible stats registered with UUID: c41cd979-1eab-475b-a42b-b8cc8e74614a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.807956045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839549597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270} {"msg":"c41cd979-1eab-475b-a42b-b8cc8e74614a active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839627263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839662127Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839713538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839741793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839766389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839790825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.83981709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839838156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839853514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839867522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44450)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.839982582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:60023)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.840056858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42756)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.840099301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.84090845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.840945744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.840971079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.840995175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.841132751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.841161946Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.84160001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.84177034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842064159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.84223088Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842259965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842274863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842298428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842313056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842326214Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842653826Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842677602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[0]R flush_numbers[0..12]: [30, 36, 31, 37, 32, 36, 30, 29, 10, 35, 29, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842695989Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[0]R generation[0..12]: [16, 20, 18, 21, 18, 20, 16, 15, 6, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842710217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842728644Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[1]R flush_numbers[0..12]: [30, 36, 31, 37, 32, 36, 30, 29, 10, 35, 29, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842742891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[1]R generation[0..12]: [16, 20, 18, 21, 18, 20, 16, 15, 6, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842756309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842770156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[2]R flush_numbers[0..12]: [30, 36, 31, 37, 32, 36, 30, 29, 10, 35, 29, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842783734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[2]R generation[0..12]: [16, 20, 18, 21, 18, 20, 16, 15, 6, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842797162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842810929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842823917Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842837315Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842850272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.84286319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"":"downstairs","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842876538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842888616Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"0","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842915051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"1","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842928209Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"client":"2","":"downstairs","session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842943286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842955774Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"c41cd979-1eab-475b-a42b-b8cc8e74614a is now active with session: b399a6c2-2506-4c74-8c58-b36d8230c7f1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842968632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.84298069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270,"session_id":"b399a6c2-2506-4c74-8c58-b36d8230c7f1"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:10:55.842993147Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 571 len 32768 data: 2 3 3 3 4 4 4 4 02/30 Read block 196 len 40960 03/30 Flush 04/30 Write block 305 len 36864 data: 3 3 3 3 3 4 4 3 3 05/30 Write block 447 len 4096 data: 5 06/30 Flush 07/30 Write block 334 len 8192 data: 3 3 08/30 Read block 64 len 28672 09/30 Read block 540 len 20480 10/30 Read block 374 len 4096 11/30 Write block 405 len 8192 data: 3 4 12/30 Read block 320 len 24576 13/30 Write block 13 len 40960 data: 5 4 5 5 6 6 6 4 4 3 14/30 Read block 423 len 12288 15/30 Write block 235 len 28672 data: 4 4 3 3 3 3 2 16/30 Read block 371 len 28672 17/30 Flush 18/30 Write block 385 len 40960 data: 2 2 2 2 2 2 2 2 2 2 19/30 Write block 381 len 16384 data: 2 2 2 2 20/30 Flush 21/30 Write block 94 len 12288 data: 3 3 4 22/30 Read block 522 len 28672 23/30 Write block 588 len 32768 data: 3 3 3 4 4 5 5 4 24/30 Read block 431 len 12288 25/30 Flush 26/30 Write block 391 len 4096 data: 3 27/30 Write block 166 len 12288 data: 4 4 5 28/30 Read block 169 len 40960 29/30 Write block 142 len 36864 data: 4 2 2 2 2 3 3 3 3 30/30 Write block 304 len 16384 data: 3 4 4 4 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:10:55.868601971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 40 40 40 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 21 21 21 37 37 37 F F F 4 080-099 22 22 22 42 42 42 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 16 16 16 30 30 30 F F F 7 140-159 15 15 15 29 29 29 T T T 8 160-179 6 6 6 10 10 10 T T T 9 180-199 20 20 20 35 35 35 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 22 22 22 39 39 39 T T T 16 320-339 22 22 22 40 40 40 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 22 22 22 41 41 41 T T T 20 400-419 22 22 22 40 40 40 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 20 20 20 36 36 36 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 22 22 22 38 38 38 F F F 29 580-599 22 22 22 42 42 42 F F F Max gen: 22, Max flush: 42 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 40 40 40 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 21 21 21 37 37 37 F F F 4 080-099 22 22 22 42 42 42 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 16 16 16 30 30 30 F F F 7 140-159 23 23 23 43 43 43 F F F 8 160-179 23 23 23 43 43 43 F F F 9 180-199 20 20 20 35 35 35 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 23 23 23 43 43 43 F F F 16 320-339 22 22 22 40 40 40 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 23 23 23 43 43 43 F F F 20 400-419 22 22 22 40 40 40 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 20 20 20 36 36 36 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 22 22 22 38 38 38 F F F 29 580-599 22 22 22 42 42 42 F F F Max gen: 23, Max flush: 43 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.197574294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.241668141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.242526961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.242565254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.242583891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.242598828Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278} {"msg":"Crucible 1e725466-eed2-4b1b-bca6-0bc07ebef43e has session id: 415082e2-fcd3-46f5-823d-ba676ce3aaaf","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.24304849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Upstairs opts: Upstairs UUID: 1e725466-eed2-4b1b-bca6-0bc07ebef43e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.24310938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Crucible stats registered with UUID: 1e725466-eed2-4b1b-bca6-0bc07ebef43e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.243134875Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274435978Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278} {"msg":"1e725466-eed2-4b1b-bca6-0bc07ebef43e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274508846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.27454253Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274590491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274619026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274651481Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274683745Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.27471548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274751633Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274786217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274820871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58704)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274861074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:34929)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.274973835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38438)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.275018707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.275545665Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.275586448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.275760798Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.275807669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.27603487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.276074823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.276393338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.276428952Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.276735618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277034106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277058442Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277073019Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277136188Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277164643Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277179801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277267526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277288622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[0]R flush_numbers[0..12]: [40, 40, 31, 37, 42, 36, 30, 43, 43, 35, 29, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277312508Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[0]R generation[0..12]: [22, 22, 18, 21, 22, 20, 16, 23, 23, 20, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277327195Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277341623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[1]R flush_numbers[0..12]: [40, 40, 31, 37, 42, 36, 30, 43, 43, 35, 29, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.27735545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[1]R generation[0..12]: [22, 22, 18, 21, 22, 20, 16, 23, 23, 20, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277369488Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277383565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[2]R flush_numbers[0..12]: [40, 40, 31, 37, 42, 36, 30, 43, 43, 35, 29, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277398533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[2]R generation[0..12]: [22, 22, 18, 21, 22, 20, 16, 23, 23, 20, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.27741648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277430907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277444085Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277457162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.27747003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277483108Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"":"downstairs","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277496336Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-09-27T17:11:00.277518472Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"0","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277532069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"1","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277545377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"client":"2","":"downstairs","session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277558785Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277571033Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"1e725466-eed2-4b1b-bca6-0bc07ebef43e is now active with session: 415082e2-fcd3-46f5-823d-ba676ce3aaaf","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277583951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277595918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278,"session_id":"415082e2-fcd3-46f5-823d-ba676ce3aaaf"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:00.277608136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 34 len 24576 data: 4 3 3 3 4 5 02/30 Read block 311 len 4096 03/30 Flush 04/30 Write block 383 len 24576 data: 3 3 3 3 3 3 05/30 Read block 574 len 24576 06/30 Read block 265 len 16384 07/30 Write block 579 len 20480 data: 4 4 4 4 5 08/30 Read block 259 len 40960 09/30 Write block 305 len 16384 data: 5 5 5 4 10/30 Read block 504 len 36864 11/30 Flush 12/30 Write block 534 len 24576 data: 4 6 5 4 4 4 13/30 Read block 545 len 12288 14/30 Write block 174 len 28672 data: 2 2 2 2 3 2 3 15/30 Write block 71 len 8192 data: 6 6 16/30 Write block 45 len 28672 data: 4 4 4 4 3 3 4 17/30 Write block 503 len 40960 data: 2 2 2 2 3 5 5 4 4 3 18/30 Write block 134 len 16384 data: 3 2 3 4 19/30 Write block 397 len 16384 data: 2 2 2 3 20/30 Read block 276 len 28672 21/30 Write block 515 len 12288 data: 5 7 5 22/30 Flush 23/30 Write block 47 len 24576 data: 5 5 4 4 5 4 24/30 Read block 332 len 32768 25/30 Read block 241 len 20480 26/30 Read block 492 len 40960 27/30 Read block 92 len 12288 28/30 Read block 206 len 20480 29/30 Read block 515 len 28672 30/30 Read block 445 len 28672 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:00.306501048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 40 40 40 F F F 1 020-039 24 24 24 44 44 44 F F F 2 040-059 24 24 24 46 46 46 T T T 3 060-079 24 24 24 46 46 46 F F F 4 080-099 22 22 22 42 42 42 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 24 24 24 46 46 46 F F F 7 140-159 23 23 23 43 43 43 F F F 8 160-179 24 24 24 46 46 46 F F F 9 180-199 24 24 24 46 46 46 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 24 24 24 45 45 45 F F F 16 320-339 22 22 22 40 40 40 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 24 24 24 46 46 46 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 24, Max flush: 46 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 40 40 40 F F F 1 020-039 24 24 24 44 44 44 F F F 2 040-059 25 25 25 47 47 47 F F F 3 060-079 24 24 24 46 46 46 F F F 4 080-099 22 22 22 42 42 42 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 24 24 24 46 46 46 F F F 7 140-159 23 23 23 43 43 43 F F F 8 160-179 24 24 24 46 46 46 F F F 9 180-199 24 24 24 46 46 46 F F F 10 200-219 15 15 15 29 29 29 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 14 14 14 28 28 28 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 24 24 24 45 45 45 F F F 16 320-339 22 22 22 40 40 40 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 24 24 24 46 46 46 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 18 18 18 32 32 32 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 25, Max flush: 47 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.576028085Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:04.620352052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.621032274Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.654317432Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.655146767Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.655184091Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.655200918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.655225243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286} {"msg":"Crucible 2558a24f-49ec-4f82-9d3e-73a0d65e0e4e has session id: 846d5f06-81b4-4f50-b8df-0cc8cebdcd8d","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.655738184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Upstairs opts: Upstairs UUID: 2558a24f-49ec-4f82-9d3e-73a0d65e0e4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.655777267Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Crucible stats registered with UUID: 2558a24f-49ec-4f82-9d3e-73a0d65e0e4e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.655793664Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687083119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286} {"msg":"2558a24f-49ec-4f82-9d3e-73a0d65e0e4e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687148448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687181033Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687228644Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687266538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687299192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687328987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687354172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687379718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687408443Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687436638Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58941)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687487249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56790)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687518584Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40806)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.687547319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.688497624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.688527068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.688541696Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.688559902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.688584548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.688605475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689186264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.68932232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689395767Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689755254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689795438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689825302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689891681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689920326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689935833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.68995308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.689967308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[0]R flush_numbers[0..12]: [40, 44, 47, 46, 42, 36, 46, 43, 46, 46, 29, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690033966Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[0]R generation[0..12]: [22, 24, 25, 24, 22, 20, 24, 23, 24, 24, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690054402Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.69006906Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[1]R flush_numbers[0..12]: [40, 44, 47, 46, 42, 36, 46, 43, 46, 46, 29, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690082757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[1]R generation[0..12]: [22, 24, 25, 24, 22, 20, 24, 23, 24, 24, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690096325Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690110123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[2]R flush_numbers[0..12]: [40, 44, 47, 46, 42, 36, 46, 43, 46, 46, 29, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.69012367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[2]R generation[0..12]: [22, 24, 25, 24, 22, 20, 24, 23, 24, 24, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690137208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690151126Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690167573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.6901811Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690194038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690206936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"":"downstairs","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:11:04.690220154Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.69024209Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"0","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690255367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"1","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690268615Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"client":"2","":"downstairs","session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690282013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690294221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"2558a24f-49ec-4f82-9d3e-73a0d65e0e4e is now active with session: 846d5f06-81b4-4f50-b8df-0cc8cebdcd8d","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690307109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690319047Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286,"session_id":"846d5f06-81b4-4f50-b8df-0cc8cebdcd8d"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:04.690331324Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 4096 data: 3 02/30 Write block 216 len 16384 data: 3 3 3 3 03/30 Write block 464 len 28672 data: 4 4 3 2 3 2 2 04/30 Write block 176 len 8192 data: 3 3 05/30 Read block 268 len 24576 06/30 Read block 311 len 28672 07/30 Read block 205 len 12288 08/30 Read block 529 len 16384 09/30 Flush 10/30 Write block 362 len 24576 data: 5 5 5 4 4 4 11/30 Read block 472 len 8192 12/30 Read block 53 len 12288 13/30 Write block 78 len 24576 data: 2 3 2 2 3 4 14/30 Write block 373 len 16384 data: 5 6 6 5 15/30 Write block 334 len 28672 data: 4 4 3 2 3 3 3 16/30 Read block 166 len 16384 17/30 Write block 202 len 12288 data: 4 4 4 18/30 Write block 182 len 36864 data: 4 4 4 5 5 4 3 3 3 19/30 Read block 145 len 4096 20/30 Read block 124 len 16384 21/30 Write block 544 len 36864 data: 2 2 2 2 3 4 5 5 5 22/30 Write block 434 len 8192 data: 2 2 23/30 Flush 24/30 Write block 277 len 28672 data: 2 2 2 2 2 3 2 25/30 Flush 26/30 Write block 186 len 28672 data: 6 5 4 4 4 2 2 27/30 Read block 549 len 20480 28/30 Write block 209 len 12288 data: 2 2 2 29/30 Read block 413 len 8192 30/30 Read block 199 len 36864 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:04.715059981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 40 40 40 F F F 1 020-039 24 24 24 44 44 44 F F F 2 040-059 25 25 25 47 47 47 F F F 3 060-079 26 26 26 49 49 49 F F F 4 080-099 26 26 26 49 49 49 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 24 24 24 46 46 46 F F F 7 140-159 23 23 23 43 43 43 F F F 8 160-179 26 26 26 48 48 48 F F F 9 180-199 26 26 26 49 49 49 T T T 10 200-219 26 26 26 49 49 49 T T T 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 26 26 26 50 50 50 F F F 14 280-299 26 26 26 50 50 50 F F F 15 300-319 24 24 24 45 45 45 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 26 26 26 49 49 49 F F F 18 360-379 26 26 26 49 49 49 F F F 19 380-399 24 24 24 46 46 46 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 26 26 26 49 49 49 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 26 26 26 48 48 48 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 26, Max flush: 50 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 40 40 40 F F F 1 020-039 24 24 24 44 44 44 F F F 2 040-059 25 25 25 47 47 47 F F F 3 060-079 26 26 26 49 49 49 F F F 4 080-099 26 26 26 49 49 49 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 24 24 24 46 46 46 F F F 7 140-159 23 23 23 43 43 43 F F F 8 160-179 26 26 26 48 48 48 F F F 9 180-199 27 27 27 51 51 51 F F F 10 200-219 27 27 27 51 51 51 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 26 26 26 50 50 50 F F F 14 280-299 26 26 26 50 50 50 F F F 15 300-319 24 24 24 45 45 45 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 26 26 26 49 49 49 F F F 18 360-379 26 26 26 49 49 49 F F F 19 380-399 24 24 24 46 46 46 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 26 26 26 49 49 49 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 26 26 26 48 48 48 F F F 24 480-499 19 19 19 34 34 34 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 27, Max flush: 51 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.018268037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.062661701Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.063687733Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.063724716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.063739764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.063754461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294} {"msg":"Crucible 07a743b8-6304-47ba-b32a-a03119925b3c has session id: a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.064192995Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Upstairs opts: Upstairs UUID: 07a743b8-6304-47ba-b32a-a03119925b3c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.064232468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Crucible stats registered with UUID: 07a743b8-6304-47ba-b32a-a03119925b3c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.064249895Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095163975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294} {"msg":"07a743b8-6304-47ba-b32a-a03119925b3c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095227294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095259719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095304451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095339645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095373239Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095407063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095441757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095475461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095513564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095784147Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52950)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095822561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37561)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095839987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41717)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.095863074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.096336451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.096366626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.096531787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.096553623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.096721034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.096792022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097111976Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097149419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097445218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097706453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097748285Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.09777931Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097796627Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097811095Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.097824492Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098063531Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098088116Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[0]R flush_numbers[0..12]: [40, 44, 47, 49, 49, 36, 46, 43, 48, 51, 51, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098105943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[0]R generation[0..12]: [22, 24, 25, 26, 26, 20, 24, 23, 26, 27, 27, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098120301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098134438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[1]R flush_numbers[0..12]: [40, 44, 47, 49, 49, 36, 46, 43, 48, 51, 51, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098148176Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[1]R generation[0..12]: [22, 24, 25, 26, 26, 20, 24, 23, 26, 27, 27, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098161803Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098175831Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[2]R flush_numbers[0..12]: [40, 44, 47, 49, 49, 36, 46, 43, 48, 51, 51, 40]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098189589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[2]R generation[0..12]: [22, 24, 25, 26, 26, 20, 24, 23, 26, 27, 27, 22]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098202936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098216784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098229602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098242639Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098255437Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098268235Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"":"downstairs","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098281713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"0","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098320536Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"1","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098333724Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"client":"2","":"downstairs","session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.09835359Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098369007Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"07a743b8-6304-47ba-b32a-a03119925b3c is now active with session: a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098382275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098394303Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:09.098406381Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 25 len 8192 data: 3 3 02/30 Write block 133 len 28672 data: 2 4 3 4 5 4 5 03/30 Read block 142 len 40960 04/30 Write block 195 len 32768 data: 3 3 3 3 3 2 2 5 05/30 Write block 540 len 36864 data: 5 4 4 3 3 3 3 3 4 06/30 Read block 442 len 40960 07/30 Read block 189 len 32768 08/30 Read block 502 len 20480 09/30 Flush 10/30 Flush 11/30 Write block 475 len 4096 data: 4 12/30 Write block 142 len 8192 data: 5 3 13/30 Read block 270 len 28672 14/30 Read block 204 len 20480 15/30 Write block 213 len 8192 data: 3 3 16/30 Read block 377 len 24576 17/30 Write block 320 len 8192 data: 2 2 18/30 Flush 19/30 Write block 397 len 4096 data: 3 20/30 Flush 21/30 Write block 10 len 24576 data: 3 3 4 6 5 6 22/30 Flush 23/30 Write block 559 len 28672 data: 2 2 3 2 2 2 3 24/30 Write block 483 len 28672 data: 2 2 2 2 2 2 2 25/30 Write block 311 len 40960 data: 5 4 4 3 3 3 3 3 3 3 26/30 Read block 535 len 8192 27/30 Write block 40 len 4096 data: 4 28/30 Write block 33 len 8192 data: 4 5 29/30 Write block 277 len 28672 data: 3 3 3 3 3 4 3 30/30 Write block 228 len 16384 data: 5 4 3 2 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 1 Done Done Sent false 28 Acked 1027 Write 2 Done Done Sent false 29 Acked 1028 Write 7 Sent Sent Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 4 8 Done 6 6 4 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.123079647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124044039Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124081473Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124108178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124133134Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12415639Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124178386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124200642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124360234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12438477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124398608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124411535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124423863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124436091Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124448359Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124460647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124476704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124489812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12450212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124514317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124526675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124539063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124551331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124563679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124576057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124588815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124601362Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.1246137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124625878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124638016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124650234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124662362Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12467443Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124688447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124700825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124712933Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124725031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124737149Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124749097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124761345Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124773552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12478564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124797808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124810096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124822334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124834502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12484686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124858988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124871006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124883043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124897241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124909519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124921657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124933655Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124945573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124957671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124969848Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124981946Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.124993994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125006162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12501823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125030358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125042336Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125054463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125066552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125078679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125090677Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125104625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125116883Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125129101Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125141308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125153436Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125165404Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125177472Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12518966Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125201898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125213946Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125226104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125238262Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12525038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125262357Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125274865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125287293Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125299751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125313418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125325776Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125338024Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125350372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12536272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125374948Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125387056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125399354Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125411891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125424169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125436207Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125448365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125460343Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125472411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125484329Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125496537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125508724Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125520892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12553459Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125546708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125558666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125570744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125582862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125594909Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125606998Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125619045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125631113Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125643241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125655429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125667597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125679705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125691733Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125703821Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125715779Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125727866Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125741624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125753832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.12576599Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125778168Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125790326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125802563Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:09.125814671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1294,"session_id":"a9fe8057-a122-40ad-bf2a-0ecc3df8dc1c"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 28 28 28 52 52 52 T T T 2 040-059 25 25 25 47 47 47 T T T 3 060-079 26 26 26 49 49 49 F F F 4 080-099 26 26 26 49 49 49 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 28 28 28 52 52 52 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 26 26 26 48 48 48 F F F 9 180-199 28 28 28 52 52 52 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 22 22 22 40 40 40 T T T 12 240-259 22 22 22 40 40 40 F F F 13 260-279 26 26 26 50 50 50 T T T 14 280-299 26 26 26 50 50 50 T T T 15 300-319 24 24 24 45 45 45 T T T 16 320-339 28 28 28 54 54 54 T T T 17 340-359 26 26 26 49 49 49 F F F 18 360-379 26 26 26 49 49 49 F F F 19 380-399 28 28 28 55 55 55 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 26 26 26 49 49 49 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 19 19 19 34 34 34 T T T 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 28 28 28 52 52 52 T T T 28 560-579 24 24 24 45 45 45 T T T 29 580-599 24 24 24 45 45 45 F F F Max gen: 28, Max flush: 56 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 29 29 29 57 57 57 F F F 3 060-079 26 26 26 49 49 49 F F F 4 080-099 26 26 26 49 49 49 F F F 5 100-119 20 20 20 36 36 36 F F F 6 120-139 28 28 28 52 52 52 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 26 26 26 48 48 48 F F F 9 180-199 28 28 28 52 52 52 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 29 29 29 57 57 57 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 29 29 29 57 57 57 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 26 26 26 49 49 49 F F F 18 360-379 26 26 26 49 49 49 F F F 19 380-399 28 28 28 55 55 55 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 26 26 26 49 49 49 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 29 29 29 57 57 57 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 29, Max flush: 57 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.481018849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.525635775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.526616114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.526653537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.526667575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.526681903Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302} {"msg":"Crucible 3899043d-ade2-4ece-b84e-8c609efa2640 has session id: e5f8e8c1-a901-4abd-94fa-b246f65ce7e4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.52721491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Upstairs opts: Upstairs UUID: 3899043d-ade2-4ece-b84e-8c609efa2640, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.527264871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Crucible stats registered with UUID: 3899043d-ade2-4ece-b84e-8c609efa2640","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.527294486Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558341203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302} {"msg":"3899043d-ade2-4ece-b84e-8c609efa2640 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558394264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558412841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558442216Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558472251Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558505035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558535899Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558568424Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558599988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558631763Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558661758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49689)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558692812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41747)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.558728586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34790)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.55876166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.559517769Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.559561221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.559602224Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.559677331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.559702316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.559756697Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.560212688Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.560414562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.560535691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.560854186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.560881881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.560895879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561077047Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561105272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.5611201Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561132677Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561146275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 57, 49, 49, 36, 52, 54, 48, 52, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561163832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[0]R generation[0..12]: [28, 29, 29, 26, 26, 20, 28, 28, 26, 28, 28, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.56117834Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561192587Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 57, 49, 49, 36, 52, 54, 48, 52, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561205995Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[1]R generation[0..12]: [28, 29, 29, 26, 26, 20, 28, 28, 26, 28, 28, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561219242Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.56123306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 57, 49, 49, 36, 52, 54, 48, 52, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561246608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[2]R generation[0..12]: [28, 29, 29, 26, 26, 20, 28, 28, 26, 28, 28, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561260145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561273943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561286701Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561299698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561312406Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561336442Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"":"downstairs","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.56134979Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561363947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"0","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561377235Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"1","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561390193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"client":"2","":"downstairs","session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.56140334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561415688Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"3899043d-ade2-4ece-b84e-8c609efa2640 is now active with session: e5f8e8c1-a901-4abd-94fa-b246f65ce7e4","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:11:13.561428566Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561441924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302,"session_id":"e5f8e8c1-a901-4abd-94fa-b246f65ce7e4"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:13.561454372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 256 len 40960 data: 5 5 4 4 5 3 3 4 4 4 02/30 Read block 332 len 32768 03/30 Read block 332 len 28672 04/30 Read block 251 len 4096 05/30 Write block 58 len 4096 data: 2 06/30 Read block 217 len 8192 07/30 Write block 87 len 40960 data: 3 2 2 2 2 2 2 4 4 5 08/30 Write block 156 len 28672 data: 2 2 2 2 3 3 3 09/30 Read block 143 len 20480 10/30 Write block 269 len 4096 data: 4 11/30 Read block 523 len 32768 12/30 Read block 204 len 40960 13/30 Read block 228 len 16384 14/30 Read block 525 len 20480 15/30 Write block 379 len 20480 data: 4 3 3 3 4 16/30 Write block 337 len 32768 data: 3 4 4 4 4 3 2 3 17/30 Write block 122 len 8192 data: 5 5 18/30 Read block 532 len 8192 19/30 Write block 187 len 4096 data: 6 20/30 Read block 450 len 4096 21/30 Read block 468 len 32768 22/30 Write block 114 len 4096 data: 3 23/30 Read block 280 len 20480 24/30 Write block 62 len 32768 data: 2 2 2 2 3 3 3 3 25/30 Read block 42 len 12288 26/30 Read block 176 len 40960 27/30 Read block 581 len 12288 28/30 Read block 474 len 12288 29/30 Read block 104 len 40960 30/30 Read block 477 len 40960 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 10 Done Done Done false 2 Acked 1001 Read 8 Done Done Done false 3 Acked 1002 Read 7 Done Done Done false 4 Acked 1003 Read 1 Done Done Done false 5 Acked 1004 Write 1 Done Done Done false 6 Acked 1005 Read 2 Done Done Done false 7 Acked 1006 Write 10 Done Done Done false 8 Acked 1007 Write 7 Done Done Done false 9 Acked 1008 Read 5 Done Done Done false 10 Acked 1009 Write 1 Done Done Done false 11 Acked 1010 Read 8 Done Done Done false 12 Acked 1011 Read 10 Done Done Done false 13 Acked 1012 Read 4 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Write 5 Done Done Done false 16 Acked 1015 Write 8 Done Done Done false 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Read 2 Done Done Done false 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:13.581971891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 29 29 29 57 57 57 T T T 3 060-079 26 26 26 49 49 49 T T T 4 080-099 26 26 26 49 49 49 T T T 5 100-119 20 20 20 36 36 36 T T T 6 120-139 28 28 28 52 52 52 T T T 7 140-159 28 28 28 54 54 54 T T T 8 160-179 26 26 26 48 48 48 T T T 9 180-199 28 28 28 52 52 52 T T T 10 200-219 28 28 28 54 54 54 F F F 11 220-239 29 29 29 57 57 57 F F F 12 240-259 22 22 22 40 40 40 T T T 13 260-279 29 29 29 57 57 57 T T T 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 29 29 29 57 57 57 T T T 17 340-359 26 26 26 49 49 49 T T T 18 360-379 26 26 26 49 49 49 T T T 19 380-399 28 28 28 55 55 55 T T T 20 400-419 24 24 24 46 46 46 F F F 21 420-439 26 26 26 49 49 49 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 29 29 29 57 57 57 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 29, Max flush: 57 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 30 30 30 58 58 58 F F F 3 060-079 30 30 30 58 58 58 F F F 4 080-099 30 30 30 58 58 58 F F F 5 100-119 30 30 30 58 58 58 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 30 30 30 58 58 58 F F F 8 160-179 30 30 30 58 58 58 F F F 9 180-199 30 30 30 58 58 58 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 29 29 29 57 57 57 F F F 12 240-259 30 30 30 58 58 58 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 30 30 30 58 58 58 F F F 17 340-359 30 30 30 58 58 58 F F F 18 360-379 30 30 30 58 58 58 F F F 19 380-399 30 30 30 58 58 58 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 26 26 26 49 49 49 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 29 29 29 57 57 57 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 30, Max flush: 58 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.015521224Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.059863568Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.060719519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.060757272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.06077263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.060787547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310} {"msg":"Crucible c53992d5-dffb-4769-bc0a-bfc5eb657ebb has session id: 8401898d-0d60-4b8d-8ed7-facc6eed14f8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.061278582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Upstairs opts: Upstairs UUID: c53992d5-dffb-4769-bc0a-bfc5eb657ebb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.061324054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Crucible stats registered with UUID: c53992d5-dffb-4769-bc0a-bfc5eb657ebb","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.061353399Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092558308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310} {"msg":"c53992d5-dffb-4769-bc0a-bfc5eb657ebb active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.09260726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092624727Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092648463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.09266297Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092676778Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092689866Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092703263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092716661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092730578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092744046Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61681)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.092995312Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39194)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.093053512Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42956)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.093092476Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094019024Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094055068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094071935Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094087382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094261642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094291507Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094776482Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.094819985Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095034667Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095430809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095456384Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095472011Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095570114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095599309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095614246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095683414Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095703751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 58, 58, 58, 58, 58, 58, 58, 58, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095725977Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[0]R generation[0..12]: [28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 28, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095740375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095754472Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 58, 58, 58, 58, 58, 58, 58, 58, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.09576816Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[1]R generation[0..12]: [28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 28, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095781748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095795725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 58, 58, 58, 58, 58, 58, 58, 58, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095809353Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[2]R generation[0..12]: [28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 28, 29]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.09582284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095836488Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Max found gen is 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095849306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Generation requested: 32 >= found:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095862433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095875611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095888679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"":"downstairs","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095901787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095916674Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"0","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095930042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"1","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095957537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"client":"2","":"downstairs","session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095970904Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.095983202Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"c53992d5-dffb-4769-bc0a-bfc5eb657ebb is now active with session: 8401898d-0d60-4b8d-8ed7-facc6eed14f8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.09599584Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.096008118Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310,"session_id":"8401898d-0d60-4b8d-8ed7-facc6eed14f8"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:18.096020346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 32768 data: 4 4 5 4 4 4 5 4 02/30 Flush 03/30 Write block 234 len 40960 data: 4 5 5 4 4 4 4 3 2 2 04/30 Write block 425 len 12288 data: 3 2 2 05/30 Read block 265 len 16384 06/30 Write block 84 len 4096 data: 5 07/30 Write block 554 len 36864 data: 3 3 3 3 2 3 3 4 3 08/30 Flush 09/30 Write block 222 len 24576 data: 3 2 3 3 4 4 10/30 Read block 232 len 32768 11/30 Read block 190 len 12288 12/30 Write block 141 len 24576 data: 4 6 4 3 3 3 13/30 Read block 498 len 28672 14/30 Read block 413 len 4096 15/30 Write block 66 len 28672 data: 4 4 4 4 6 7 7 16/30 Read block 585 len 32768 17/30 Read block 247 len 40960 18/30 Read block 361 len 40960 19/30 Write block 545 len 24576 data: 4 4 4 5 5 6 20/30 Flush 21/30 Write block 55 len 36864 data: 4 3 3 3 2 2 2 3 3 22/30 Read block 28 len 8192 23/30 Write block 83 len 32768 data: 5 6 4 4 4 3 3 3 24/30 Read block 409 len 12288 25/30 Read block 212 len 28672 26/30 Read block 366 len 16384 27/30 Read block 566 len 24576 28/30 Read block 233 len 8192 29/30 Read block 554 len 32768 30/30 Read block 343 len 40960 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 9 9 10 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:18.123929709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 30 30 30 58 58 58 T T T 3 060-079 32 32 32 61 61 61 T T T 4 080-099 32 32 32 60 60 60 T T T 5 100-119 30 30 30 58 58 58 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 32 32 32 61 61 61 F F F 8 160-179 30 30 30 58 58 58 F F F 9 180-199 30 30 30 58 58 58 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 32 32 32 61 61 61 F F F 12 240-259 32 32 32 60 60 60 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 30 30 30 58 58 58 F F F 17 340-359 30 30 30 58 58 58 F F F 18 360-379 32 32 32 59 59 59 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 32 32 32 60 60 60 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 32 32 32 61 61 61 F F F 28 560-579 32 32 32 60 60 60 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 32, Max flush: 61 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 33 33 33 62 62 62 F F F 3 060-079 33 33 33 62 62 62 F F F 4 080-099 33 33 33 62 62 62 F F F 5 100-119 30 30 30 58 58 58 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 32 32 32 61 61 61 F F F 8 160-179 30 30 30 58 58 58 F F F 9 180-199 30 30 30 58 58 58 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 32 32 32 61 61 61 F F F 12 240-259 32 32 32 60 60 60 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 30 30 30 58 58 58 F F F 17 340-359 30 30 30 58 58 58 F F F 18 360-379 32 32 32 59 59 59 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 24 24 24 46 46 46 F F F 21 420-439 32 32 32 60 60 60 F F F 22 440-459 22 22 22 39 39 39 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 24 24 24 46 46 46 F F F 26 520-539 24 24 24 46 46 46 F F F 27 540-559 32 32 32 61 61 61 F F F 28 560-579 32 32 32 60 60 60 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 33, Max flush: 62 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.427146534Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:22.470847809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.47147026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.504207983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.505083351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.505131243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.50514883Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.505166916Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318} {"msg":"Crucible b833be88-12be-4945-a444-b80c6b123be1 has session id: 4c3d8a8b-e859-4b0d-9585-4cfc7da27173","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.505558898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Upstairs opts: Upstairs UUID: b833be88-12be-4945-a444-b80c6b123be1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.505603141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Crucible stats registered with UUID: b833be88-12be-4945-a444-b80c6b123be1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.505623587Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536572451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318} {"msg":"b833be88-12be-4945-a444-b80c6b123be1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536628372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536656137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536687261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536705798Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536722975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536739862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536756759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536773926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536839705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536874949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_connection connected from Ok(127.0.0.1:54759)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.536907383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_connection connected from Ok(127.0.0.1:42965)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.537023273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_connection connected from Ok(127.0.0.1:33337)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.537069875Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.537916217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.537948082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.537971718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.537993564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.53813185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.538226523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.538692502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.538723967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.538830278Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539332901Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539364165Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539379983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539392791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539406428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539419596Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539431744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539444882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 62, 62, 62, 58, 58, 61, 58, 58, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539475286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[0]R generation[0..12]: [28, 29, 33, 33, 33, 30, 30, 32, 30, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539491294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.5395092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 62, 62, 62, 58, 58, 61, 58, 58, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539523898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[1]R generation[0..12]: [28, 29, 33, 33, 33, 30, 30, 32, 30, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539538095Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539552193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 62, 62, 62, 58, 58, 61, 58, 58, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.53956577Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[2]R generation[0..12]: [28, 29, 33, 33, 33, 30, 30, 32, 30, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539579468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539593556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539606453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539619581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539632379Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539645127Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"":"downstairs","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539658484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539670542Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"0","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539697307Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"1","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539722623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"client":"2","":"downstairs","session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539751318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539776104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"b833be88-12be-4945-a444-b80c6b123be1 is now active with session: 4c3d8a8b-e859-4b0d-9585-4cfc7da27173","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.53980004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539822906Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318,"session_id":"4c3d8a8b-e859-4b0d-9585-4cfc7da27173"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:22.539847001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 20480 data: 4 4 5 5 5 02/30 Write block 324 len 40960 data: 3 3 3 3 4 3 3 3 3 3 03/30 Read block 196 len 12288 04/30 Write block 162 len 8192 data: 4 3 05/30 Read block 554 len 4096 06/30 Read block 192 len 32768 07/30 Write block 364 len 40960 data: 6 5 5 5 3 3 4 5 5 6 08/30 Write block 439 len 36864 data: 2 2 2 2 2 2 2 5 6 09/30 Read block 425 len 40960 10/30 Read block 397 len 24576 11/30 Write block 402 len 12288 data: 3 3 3 12/30 Write block 529 len 32768 data: 6 8 7 6 5 5 7 6 13/30 Write block 2 len 32768 data: 2 2 4 4 4 4 3 3 14/30 Write block 168 len 28672 data: 6 4 3 3 3 3 3 15/30 Read block 137 len 20480 16/30 Write block 41 len 28672 data: 5 5 5 4 5 5 6 17/30 Read block 10 len 36864 18/30 Write block 547 len 32768 data: 5 6 6 7 6 6 5 4 19/30 Read block 483 len 24576 20/30 Read block 133 len 20480 21/30 Write block 537 len 40960 data: 5 5 5 6 5 5 4 4 5 5 22/30 Read block 428 len 4096 23/30 Read block 53 len 28672 24/30 Flush 25/30 Write block 5 len 16384 data: 5 5 5 4 26/30 Read block 357 len 8192 27/30 Read block 462 len 32768 28/30 Write block 517 len 40960 data: 6 4 4 4 4 5 4 5 4 2 29/30 Read block 567 len 40960 30/30 Write block 79 len 12288 data: 4 3 3 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:22.565248961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 63 63 63 T T T 1 020-039 29 29 29 57 57 57 F F F 2 040-059 34 34 34 63 63 63 F F F 3 060-079 33 33 33 62 62 62 T T T 4 080-099 33 33 33 62 62 62 T T T 5 100-119 30 30 30 58 58 58 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 32 32 32 61 61 61 F F F 8 160-179 34 34 34 63 63 63 F F F 9 180-199 30 30 30 58 58 58 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 32 32 32 61 61 61 F F F 12 240-259 32 32 32 60 60 60 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 34 34 34 63 63 63 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 34 34 34 63 63 63 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 34 34 34 63 63 63 F F F 21 420-439 34 34 34 63 63 63 F F F 22 440-459 34 34 34 63 63 63 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 24 24 24 46 46 46 T T T 26 520-539 34 34 34 63 63 63 T T T 27 540-559 34 34 34 63 63 63 F F F 28 560-579 32 32 32 60 60 60 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 34, Max flush: 63 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 64 64 64 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 34 34 34 63 63 63 F F F 3 060-079 35 35 35 64 64 64 F F F 4 080-099 35 35 35 64 64 64 F F F 5 100-119 30 30 30 58 58 58 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 32 32 32 61 61 61 F F F 8 160-179 34 34 34 63 63 63 F F F 9 180-199 30 30 30 58 58 58 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 32 32 32 61 61 61 F F F 12 240-259 32 32 32 60 60 60 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 34 34 34 63 63 63 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 34 34 34 63 63 63 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 34 34 34 63 63 63 F F F 21 420-439 34 34 34 63 63 63 F F F 22 440-459 34 34 34 63 63 63 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 35 35 35 64 64 64 F F F 26 520-539 35 35 35 64 64 64 F F F 27 540-559 34 34 34 63 63 63 F F F 28 560-579 32 32 32 60 60 60 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 35, Max flush: 64 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.925606791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.9703808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.971283722Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.971317427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.971331224Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.971345582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326} {"msg":"Crucible 3600473c-fdf3-40de-879b-b57c1b524c50 has session id: 19cf38d9-7e9a-400c-a315-33eb880477b9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.971862222Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Upstairs opts: Upstairs UUID: 3600473c-fdf3-40de-879b-b57c1b524c50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.971913003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Crucible stats registered with UUID: 3600473c-fdf3-40de-879b-b57c1b524c50","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:26.971939119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003288983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326} {"msg":"3600473c-fdf3-40de-879b-b57c1b524c50 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003344663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.00336503Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003394805Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003413071Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003430708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003447505Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003470461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003487338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003504535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003521392Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:43941)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.00364967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62037)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003741524Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44121)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.003816331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.004429665Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.004462569Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.004599955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.004663654Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.004771865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.00479836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005247812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005279266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.00554619Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005841649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005882332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005912756Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005936492Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005962928Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.005987603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006095724Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.00612396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[0]R flush_numbers[0..12]: [64, 57, 63, 64, 64, 58, 58, 61, 63, 58, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006139197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[0]R generation[0..12]: [35, 29, 34, 35, 35, 30, 30, 32, 34, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006153565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006167932Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[1]R flush_numbers[0..12]: [64, 57, 63, 64, 64, 58, 58, 61, 63, 58, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006181809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[1]R generation[0..12]: [35, 29, 34, 35, 35, 30, 30, 32, 34, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006195367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006209395Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[2]R flush_numbers[0..12]: [64, 57, 63, 64, 64, 58, 58, 61, 63, 58, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006227082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[2]R generation[0..12]: [35, 29, 34, 35, 35, 30, 30, 32, 34, 30, 28, 32]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006241069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006255097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006268154Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006281252Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.00629524Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006308467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"":"downstairs","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006321915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006334043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":Wait for a query_work_queue command to finish before sending IO 1326,"client":"0","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006395832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"1","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006412409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"client":"2","":"downstairs","session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006425997Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006438205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"3600473c-fdf3-40de-879b-b57c1b524c50 is now active with session: 19cf38d9-7e9a-400c-a315-33eb880477b9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006450963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006462821Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326,"session_id":"19cf38d9-7e9a-400c-a315-33eb880477b9"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:27.006477668Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 188 len 36864 data: 5 5 5 3 3 2 3 4 4 02/30 Read block 534 len 16384 03/30 Read block 526 len 12288 04/30 Read block 214 len 28672 05/30 Read block 142 len 32768 06/30 Write block 11 len 36864 data: 4 5 7 6 7 6 7 7 7 07/30 Read block 282 len 8192 08/30 Read block 386 len 12288 09/30 Write block 520 len 40960 data: 5 5 6 5 6 5 3 2 3 7 10/30 Write block 229 len 16384 data: 5 4 3 2 11/30 Read block 216 len 12288 12/30 Read block 407 len 32768 13/30 Flush 14/30 Flush 15/30 Write block 52 len 12288 data: 5 4 5 16/30 Read block 511 len 36864 17/30 Flush 18/30 Write block 103 len 40960 data: 5 4 3 5 4 4 4 4 4 4 19/30 Write block 48 len 8192 data: 6 5 20/30 Read block 12 len 20480 21/30 Write block 523 len 12288 data: 6 7 6 22/30 Read block 502 len 36864 23/30 Write block 103 len 24576 data: 6 5 4 6 5 5 24/30 Read block 198 len 16384 25/30 Read block 499 len 28672 26/30 Read block 170 len 8192 27/30 Read block 404 len 32768 28/30 Read block 358 len 24576 29/30 Read block 197 len 24576 30/30 Read block 227 len 16384 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 10 Done Done Done false 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 12 13 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:27.032563069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 65 65 65 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 36 36 36 67 67 67 T T T 3 060-079 35 35 35 64 64 64 F F F 4 080-099 35 35 35 64 64 64 F F F 5 100-119 30 30 30 58 58 58 T T T 6 120-139 30 30 30 58 58 58 F F F 7 140-159 32 32 32 61 61 61 F F F 8 160-179 34 34 34 63 63 63 F F F 9 180-199 36 36 36 65 65 65 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 36 36 36 65 65 65 F F F 12 240-259 32 32 32 60 60 60 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 34 34 34 63 63 63 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 34 34 34 63 63 63 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 34 34 34 63 63 63 F F F 21 420-439 34 34 34 63 63 63 F F F 22 440-459 34 34 34 63 63 63 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 35 35 35 64 64 64 F F F 26 520-539 36 36 36 65 65 65 T T T 27 540-559 34 34 34 63 63 63 F F F 28 560-579 32 32 32 60 60 60 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 36, Max flush: 67 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 65 65 65 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 37 37 37 68 68 68 F F F 3 060-079 35 35 35 64 64 64 F F F 4 080-099 35 35 35 64 64 64 F F F 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 32 32 32 61 61 61 F F F 8 160-179 34 34 34 63 63 63 F F F 9 180-199 36 36 36 65 65 65 F F F 10 200-219 28 28 28 54 54 54 F F F 11 220-239 36 36 36 65 65 65 F F F 12 240-259 32 32 32 60 60 60 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 34 34 34 63 63 63 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 34 34 34 63 63 63 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 34 34 34 63 63 63 F F F 21 420-439 34 34 34 63 63 63 F F F 22 440-459 34 34 34 63 63 63 F F F 23 460-479 28 28 28 54 54 54 F F F 24 480-499 29 29 29 57 57 57 F F F 25 500-519 35 35 35 64 64 64 F F F 26 520-539 37 37 37 68 68 68 F F F 27 540-559 34 34 34 63 63 63 F F F 28 560-579 32 32 32 60 60 60 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 37, Max flush: 68 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.331369591Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:31.375308774Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.375894052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.407670523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.40855028Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.408590153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.40860891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.408624237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334} {"msg":"Crucible 9d1423ea-ee5e-412f-81e8-99064cd7ead8 has session id: 7cca544b-0c0e-4f25-94ff-c68bc74c9105","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.40918234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Upstairs opts: Upstairs UUID: 9d1423ea-ee5e-412f-81e8-99064cd7ead8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.409231641Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Crucible stats registered with UUID: 9d1423ea-ee5e-412f-81e8-99064cd7ead8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.409252707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439571322Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334} {"msg":"9d1423ea-ee5e-412f-81e8-99064cd7ead8 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439608815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439627402Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439656956Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439675323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.43969295Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439709887Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439726644Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439746771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439785034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439815189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_connection connected from Ok(127.0.0.1:37068)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439844174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_connection connected from Ok(127.0.0.1:35503)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439876048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_connection connected from Ok(127.0.0.1:55945)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.439904724Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.440715192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.440745827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.440760835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.440774642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.440929305Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.440996613Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.441467192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.441492197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.44153497Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442073896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442113669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442134825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442208812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442228319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442242057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442313684Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442334071Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[0]R flush_numbers[0..12]: [65, 57, 68, 64, 64, 68, 58, 61, 63, 65, 54, 65]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442359106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[0]R generation[0..12]: [36, 29, 37, 35, 35, 37, 30, 32, 34, 36, 28, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442374454Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442388711Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[1]R flush_numbers[0..12]: [65, 57, 68, 64, 64, 68, 58, 61, 63, 65, 54, 65]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442402329Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[1]R generation[0..12]: [36, 29, 37, 35, 35, 37, 30, 32, 34, 36, 28, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442415616Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442429334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[2]R flush_numbers[0..12]: [65, 57, 68, 64, 64, 68, 58, 61, 63, 65, 54, 65]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442442702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[2]R generation[0..12]: [36, 29, 37, 35, 35, 37, 30, 32, 34, 36, 28, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.4424561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442473137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442486454Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442499562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.44251227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442524888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"":"downstairs","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442537935Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442549713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"0","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442573459Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"1","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442586497Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"client":"2","":"downstairs","session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442599744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442611892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"9d1423ea-ee5e-412f-81e8-99064cd7ead8 is now active with session: 7cca544b-0c0e-4f25-94ff-c68bc74c9105","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.44262463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442636588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334,"session_id":"7cca544b-0c0e-4f25-94ff-c68bc74c9105"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:31.442648596Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 309 len 8192 data: 4 5 02/30 Read block 496 len 24576 03/30 Read block 483 len 4096 04/30 Read block 345 len 12288 05/30 Write block 522 len 16384 data: 7 7 8 7 06/30 Write block 453 len 32768 data: 4 3 2 2 2 2 2 2 07/30 Write block 558 len 32768 data: 3 4 4 5 4 3 3 4 08/30 Read block 98 len 20480 09/30 Read block 339 len 12288 10/30 Write block 479 len 8192 data: 2 2 11/30 Read block 16 len 32768 12/30 Write block 194 len 32768 data: 4 5 5 4 4 4 3 3 13/30 Read block 229 len 28672 14/30 Read block 262 len 8192 15/30 Read block 411 len 12288 16/30 Read block 454 len 12288 17/30 Flush 18/30 Write block 57 len 8192 data: 4 4 19/30 Read block 279 len 28672 20/30 Write block 235 len 4096 data: 6 21/30 Write block 362 len 12288 data: 6 6 7 22/30 Write block 152 len 36864 data: 3 3 3 3 3 3 3 3 4 23/30 Write block 273 len 8192 data: 3 2 24/30 Read block 287 len 40960 25/30 Write block 254 len 12288 data: 4 4 6 26/30 Read block 3 len 20480 27/30 Read block 293 len 16384 28/30 Write block 544 len 36864 data: 5 6 6 6 7 7 8 7 7 29/30 Write block 582 len 16384 data: 5 6 3 4 30/30 Read block 331 len 20480 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Read 7 Done Done Done false 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 9 Done Done Sent false 29 Acked 1028 Write 4 Done Done Sent false 30 Acked 1029 Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 3 4 Done 12 13 10 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:31.464200485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 65 65 65 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 37 37 37 68 68 68 T T T 3 060-079 35 35 35 64 64 64 F F F 4 080-099 35 35 35 64 64 64 F F F 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 32 32 32 61 61 61 T T T 8 160-179 34 34 34 63 63 63 T T T 9 180-199 38 38 38 69 69 69 F F F 10 200-219 38 38 38 69 69 69 F F F 11 220-239 36 36 36 65 65 65 T T T 12 240-259 32 32 32 60 60 60 T T T 13 260-279 30 30 30 58 58 58 T T T 14 280-299 29 29 29 57 57 57 F F F 15 300-319 38 38 38 69 69 69 F F F 16 320-339 34 34 34 63 63 63 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 34 34 34 63 63 63 T T T 19 380-399 32 32 32 59 59 59 F F F 20 400-419 34 34 34 63 63 63 F F F 21 420-439 34 34 34 63 63 63 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 38 38 38 69 69 69 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 35 35 35 64 64 64 F F F 26 520-539 38 38 38 69 69 69 F F F 27 540-559 38 38 38 69 69 69 T T T 28 560-579 38 38 38 69 69 69 F F F 29 580-599 24 24 24 45 45 45 T T T Max gen: 38, Max flush: 69 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 65 65 65 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 35 35 35 64 64 64 F F F 4 080-099 35 35 35 64 64 64 F F F 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 39 39 39 70 70 70 F F F 8 160-179 39 39 39 70 70 70 F F F 9 180-199 38 38 38 69 69 69 F F F 10 200-219 38 38 38 69 69 69 F F F 11 220-239 39 39 39 70 70 70 F F F 12 240-259 39 39 39 70 70 70 F F F 13 260-279 39 39 39 70 70 70 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 38 38 38 69 69 69 F F F 16 320-339 34 34 34 63 63 63 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 39 39 39 70 70 70 F F F 19 380-399 32 32 32 59 59 59 F F F 20 400-419 34 34 34 63 63 63 F F F 21 420-439 34 34 34 63 63 63 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 38 38 38 69 69 69 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 35 35 35 64 64 64 F F F 26 520-539 38 38 38 69 69 69 F F F 27 540-559 39 39 39 70 70 70 F F F 28 560-579 38 38 38 69 69 69 F F F 29 580-599 39 39 39 70 70 70 F F F Max gen: 39, Max flush: 70 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:35.97279425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.018966585Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.019881786Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.019917419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.019931687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.019946354Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342} {"msg":"Crucible 6ec26be7-7408-4cae-a9cd-b474b7fc48ad has session id: 7980369d-1330-45eb-897c-c92c28b46ffe","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.020421602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Upstairs opts: Upstairs UUID: 6ec26be7-7408-4cae-a9cd-b474b7fc48ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.020477712Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Crucible stats registered with UUID: 6ec26be7-7408-4cae-a9cd-b474b7fc48ad","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.020497418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051816408Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342} {"msg":"6ec26be7-7408-4cae-a9cd-b474b7fc48ad active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051872199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051889806Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051917591Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051935128Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051952455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051969092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.051985849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.052002196Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.052019683Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.05203686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:33370)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.052128804Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:46248)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.052166327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:61542)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.05226557Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.053133889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.053166553Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.05318325Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.053197128Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.053338883Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.053374877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.053834017Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.05387506Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054028274Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054402568Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054427934Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054442921Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054488323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054519368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054549553Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054649945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054669622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[0]R flush_numbers[0..12]: [65, 57, 70, 64, 64, 68, 58, 70, 70, 69, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054692758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[0]R generation[0..12]: [36, 29, 39, 35, 35, 37, 30, 39, 39, 38, 38, 39]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054708105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054722663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[1]R flush_numbers[0..12]: [65, 57, 70, 64, 64, 68, 58, 70, 70, 69, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.05473631Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[1]R generation[0..12]: [36, 29, 39, 35, 35, 37, 30, 39, 39, 38, 38, 39]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054750038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054763975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[2]R flush_numbers[0..12]: [65, 57, 70, 64, 64, 68, 58, 70, 70, 69, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054777523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[2]R generation[0..12]: [36, 29, 39, 35, 35, 37, 30, 39, 39, 38, 38, 39]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054791151Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054804948Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054817836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054831044Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054846631Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054859749Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"":"downstairs","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054873136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054885304Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,Wait for a query_work_queue command to finish before sending IO "client":"0","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.05490963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"1","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054922738Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"client":"2","":"downstairs","session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054935885Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054948043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"6ec26be7-7408-4cae-a9cd-b474b7fc48ad is now active with session: 7980369d-1330-45eb-897c-c92c28b46ffe","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054961021Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054973049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342,"session_id":"7980369d-1330-45eb-897c-c92c28b46ffe"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:36.054985297Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 24576 data: 4 3 3 3 2 2 02/30 Write block 574 len 32768 data: 4 5 5 5 5 5 5 5 03/30 Write block 414 len 8192 data: 5 6 04/30 Read block 184 len 24576 05/30 Flush 06/30 Flush 07/30 Write block 171 len 8192 data: 4 4 08/30 Write block 537 len 20480 data: 6 6 6 7 6 09/30 Read block 117 len 24576 10/30 Read block 274 len 8192 11/30 Flush 12/30 Write block 569 len 20480 data: 3 2 3 4 4 13/30 Read block 553 len 12288 14/30 Flush 15/30 Write block 423 len 8192 data: 3 3 16/30 Read block 109 len 4096 17/30 Read block 86 len 40960 18/30 Write block 146 len 40960 data: 4 4 4 4 4 3 4 4 4 4 19/30 Read block 383 len 40960 20/30 Read block 373 len 40960 21/30 Read block 298 len 16384 22/30 Read block 190 len 24576 23/30 Write block 7 len 32768 data: 6 5 4 4 5 6 8 7 24/30 Read block 416 len 20480 25/30 Flush 26/30 Write block 314 len 32768 data: 4 4 4 4 4 4 4 3 27/30 Read block 540 len 20480 28/30 Write block 253 len 36864 data: 3 5 5 7 6 5 5 6 4 29/30 Write block 72 len 40960 data: 8 5 5 5 5 3 3 5 4 4 30/30 Read block 329 len 24576 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:36.082069714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 75 75 75 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 35 35 35 64 64 64 T T T 4 080-099 35 35 35 64 64 64 T T T 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 40 40 40 73 73 73 F F F 9 180-199 38 38 38 69 69 69 F F F 10 200-219 38 38 38 69 69 69 F F F 11 220-239 39 39 39 70 70 70 F F F 12 240-259 39 39 39 70 70 70 T T T 13 260-279 39 39 39 70 70 70 T T T 14 280-299 29 29 29 57 57 57 F F F 15 300-319 38 38 38 69 69 69 T T T 16 320-339 34 34 34 63 63 63 T T T 17 340-359 34 34 34 63 63 63 F F F 18 360-379 39 39 39 70 70 70 F F F 19 380-399 40 40 40 71 71 71 F F F 20 400-419 40 40 40 71 71 71 F F F 21 420-439 40 40 40 75 75 75 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 38 38 38 69 69 69 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 35 35 35 64 64 64 F F F 26 520-539 40 40 40 73 73 73 F F F 27 540-559 40 40 40 73 73 73 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 40 40 40 71 71 71 F F F Max gen: 40, Max flush: 75 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 75 75 75 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 41 41 41 76 76 76 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 40 40 40 73 73 73 F F F 9 180-199 38 38 38 69 69 69 F F F 10 200-219 38 38 38 69 69 69 F F F 11 220-239 39 39 39 70 70 70 F F F 12 240-259 41 41 41 76 76 76 F F F 13 260-279 41 41 41 76 76 76 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 41 41 41 76 76 76 F F F 16 320-339 41 41 41 76 76 76 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 39 39 39 70 70 70 F F F 19 380-399 40 40 40 71 71 71 F F F 20 400-419 40 40 40 71 71 71 F F F 21 420-439 40 40 40 75 75 75 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 38 38 38 69 69 69 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 35 35 35 64 64 64 F F F 26 520-539 40 40 40 73 73 73 F F F 27 540-559 40 40 40 73 73 73 F F F 28 560-579 40 40 40 74 74 74 F F F 29 580-599 40 40 40 71 71 71 F F F Max gen: 41, Max flush: 76 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.401818461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.445735909Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.44659167Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.446633273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.44665093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.446668716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350} {"msg":"Crucible 24c902d9-2e4f-4296-a21b-fd478c81ea03 has session id: 7574f85f-b8a3-4704-8015-eebcb6fc7dfd","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.447145093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Upstairs opts: Upstairs UUID: 24c902d9-2e4f-4296-a21b-fd478c81ea03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.447193485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Crucible stats registered with UUID: 24c902d9-2e4f-4296-a21b-fd478c81ea03","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.44722291Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478130301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350} {"msg":"24c902d9-2e4f-4296-a21b-fd478c81ea03 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478185672Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478212537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478249851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478284125Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478318089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478351333Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478385127Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478418281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478452815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478488089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44793)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478520833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47129)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478556977Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61193)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.478589501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.479418347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.479452082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.479467679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.479481556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.479730963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.479758868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.480315281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.480343406Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.48049619Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.48095546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.480981975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.48101143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.48106954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481097175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481111982Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.48112453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481138058Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[0]R flush_numbers[0..12]: [75, 57, 70, 76, 76, 68, 58, 75, 73, 69, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481199657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[0]R generation[0..12]: [40, 29, 39, 41, 41, 37, 30, 40, 40, 38, 38, 39]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481219714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481238491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[1]R flush_numbers[0..12]: [75, 57, 70, 76, 76, 68, 58, 75, 73, 69, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481252758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[1]R generation[0..12]: [40, 29, 39, 41, 41, 37, 30, 40, 40, 38, 38, 39]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481266306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481280153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[2]R flush_numbers[0..12]: [75, 57, 70, 76, 76, 68, 58, 75, 73, 69, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481293451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[2]R generation[0..12]: [40, 29, 39, 41, 41, 37, 30, 40, 40, 38, 38, 39]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481306789Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481320446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481333184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481346262Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481359219Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481372157Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"":"downstairs","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"No downstairs reconciliation required","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481395233Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481407351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"0","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481420509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"1","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481434197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"client":"2","":"downstairs","session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481449084Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481461382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"24c902d9-2e4f-4296-a21b-fd478c81ea03 is now active with session: 7574f85f-b8a3-4704-8015-eebcb6fc7dfd","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.48147427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481486317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350,"session_id":"7574f85f-b8a3-4704-8015-eebcb6fc7dfd"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:40.481498545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 413 len 4096 data: 5 02/30 Read block 224 len 36864 03/30 Read block 475 len 40960 04/30 Read block 468 len 32768 05/30 Flush 06/30 Write block 6 len 32768 data: 6 7 6 5 5 6 7 9 07/30 Read block 283 len 28672 08/30 Write block 590 len 16384 data: 4 5 5 6 09/30 Read block 131 len 28672 10/30 Write block 556 len 28672 data: 4 4 4 5 5 6 5 11/30 Read block 104 len 28672 12/30 Write block 548 len 36864 data: 8 8 9 8 8 6 5 4 5 13/30 Write block 373 len 8192 data: 7 7 14/30 Write block 5 len 32768 data: 6 7 8 7 6 6 7 8 15/30 Flush 16/30 Flush 17/30 Write block 302 len 20480 data: 2 2 4 6 6 18/30 Write block 425 len 20480 data: 4 3 3 3 2 19/30 Read block 206 len 20480 20/30 Write block 515 len 12288 data: 6 8 7 21/30 Write block 361 len 24576 data: 4 7 7 8 6 6 22/30 Write block 97 len 4096 data: 4 23/30 Write block 574 len 8192 data: 5 6 24/30 Read block 89 len 40960 25/30 Write block 230 len 36864 data: 5 4 3 4 5 7 6 5 5 26/30 Read block 474 len 8192 27/30 Read block 174 len 16384 28/30 Write block 239 len 4096 data: 5 29/30 Read block 230 len 16384 30/30 Read block 388 len 40960 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:40.505452807Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 41 41 41 76 76 76 F F F 4 080-099 41 41 41 76 76 76 T T T 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 40 40 40 73 73 73 F F F 9 180-199 38 38 38 69 69 69 F F F 10 200-219 38 38 38 69 69 69 F F F 11 220-239 39 39 39 70 70 70 T T T 12 240-259 41 41 41 76 76 76 F F F 13 260-279 41 41 41 76 76 76 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 41 41 41 76 76 76 T T T 16 320-339 41 41 41 76 76 76 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 42 42 42 78 78 78 T T T 19 380-399 40 40 40 71 71 71 F F F 20 400-419 42 42 42 77 77 77 F F F 21 420-439 40 40 40 75 75 75 T T T 22 440-459 38 38 38 69 69 69 F F F 23 460-479 38 38 38 69 69 69 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 35 35 35 64 64 64 T T T 26 520-539 40 40 40 73 73 73 F F F 27 540-559 42 42 42 78 78 78 F F F 28 560-579 42 42 42 78 78 78 T T T 29 580-599 42 42 42 78 78 78 F F F Max gen: 42, Max flush: 78 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 78 78 78 F F F 1 020-039 29 29 29 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 41 41 41 76 76 76 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 40 40 40 73 73 73 F F F 9 180-199 38 38 38 69 69 69 F F F 10 200-219 38 38 38 69 69 69 F F F 11 220-239 43 43 43 79 79 79 F F F 12 240-259 41 41 41 76 76 76 F F F 13 260-279 41 41 41 76 76 76 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 41 41 41 76 76 76 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 43 43 43 79 79 79 F F F 19 380-399 40 40 40 71 71 71 F F F 20 400-419 42 42 42 77 77 77 F F F 21 420-439 43 43 43 79 79 79 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 38 38 38 69 69 69 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 43 43 43 79 79 79 F F F 26 520-539 40 40 40 73 73 73 F F F 27 540-559 42 42 42 78 78 78 F F F 28 560-579 43 43 43 79 79 79 F F F 29 580-599 42 42 42 78 78 78 F F F Max gen: 43, Max flush: 79 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.843140152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.887274462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.888233105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.888278187Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.888295644Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.888313561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358} {"msg":"Crucible a2168842-390c-46d7-bb29-c00a01c88827 has session id: 4a445c90-6618-4369-b494-6138bfa62d71","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.888873794Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Upstairs opts: Upstairs UUID: a2168842-390c-46d7-bb29-c00a01c88827, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.888920835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Crucible stats registered with UUID: a2168842-390c-46d7-bb29-c00a01c88827","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.888946571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920042419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358} {"msg":"a2168842-390c-46d7-bb29-c00a01c88827 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920100799Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920127865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920178926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920216459Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920254382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920290716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920317322Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920349356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.92038144Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920416604Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_connection connected from Ok(127.0.0.1:53819)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920449629Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_connection connected from Ok(127.0.0.1:43451)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920486502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_connection connected from Ok(127.0.0.1:46134)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.920537203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.921349062Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.921384896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.921401463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.9214153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.921750772Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.921775408Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.922243296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.922290988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.92233898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.922847991Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.922885535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.92290986Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.922934056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.922958462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.922972809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923036878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923056915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[0]R flush_numbers[0..12]: [78, 57, 70, 76, 79, 68, 58, 75, 73, 69, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923081071Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[0]R generation[0..12]: [42, 29, 39, 41, 43, 37, 30, 40, 40, 38, 38, 43]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923095348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923109536Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[1]R flush_numbers[0..12]: [78, 57, 70, 76, 79, 68, 58, 75, 73, 69, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923123173Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[1]R generation[0..12]: [42, 29, 39, 41, 43, 37, 30, 40, 40, 38, 38, 43]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923136721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923154648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[2]R flush_numbers[0..12]: [78, 57, 70, 76, 79, 68, 58, 75, 73, 69, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923168426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[2]R generation[0..12]: [42, 29, 39, 41, 43, 37, 30, 40, 40, 38, 38, 43]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923181943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923195621Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923208268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923221266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923233964Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923247172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"":"downstairs","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.92326065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923272687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"0","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:11:44.923285975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"1","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923307161Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"client":"2","":"downstairs","session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923320359Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923339726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"a2168842-390c-46d7-bb29-c00a01c88827 is now active with session: 4a445c90-6618-4369-b494-6138bfa62d71","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923354963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.92337011Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358,"session_id":"4a445c90-6618-4369-b494-6138bfa62d71"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:44.923382878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 36864 data: 5 4 4 5 4 3 3 3 3 02/30 Write block 206 len 12288 data: 3 3 2 03/30 Write block 314 len 12288 data: 5 5 5 04/30 Write block 372 len 4096 data: 6 05/30 Read block 408 len 20480 06/30 Write block 322 len 24576 data: 2 2 4 4 4 4 07/30 Read block 64 len 40960 08/30 Read block 456 len 16384 09/30 Write block 516 len 32768 data: 9 8 5 5 6 6 8 8 10/30 Write block 513 len 28672 data: 4 4 7 10 9 6 6 11/30 Read block 245 len 28672 12/30 Read block 406 len 40960 13/30 Write block 398 len 20480 data: 3 3 4 3 4 14/30 Flush 15/30 Write block 173 len 4096 data: 4 16/30 Write block 15 len 32768 data: 8 7 8 8 8 5 5 4 17/30 Read block 452 len 16384 18/30 Flush 19/30 Write block 428 len 32768 data: 4 3 2 2 2 2 3 3 20/30 Read block 167 len 20480 21/30 Write block 275 len 8192 data: 2 3 22/30 Write block 512 len 28672 data: 4 5 5 8 11 10 7 23/30 Write block 381 len 36864 data: 5 5 6 5 4 4 4 4 3 24/30 Read block 334 len 36864 25/30 Flush 26/30 Write block 463 len 24576 data: 4 5 5 4 3 4 27/30 Write block 383 len 36864 data: 7 6 5 5 5 5 4 3 5 28/30 Write block 588 len 20480 data: 4 4 5 6 6 29/30 Read block 525 len 28672 30/30 Write block 521 len 8192 data: 7 9 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 9 Sent Sent Done false 28 Acked 1027 Write 5 Sent Sent Done false 29 Acked 1028 Read 7 Sent Sent Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 1 9 Done 1 1 4 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:44.950637585Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 81 81 81 F F F 1 020-039 44 44 44 81 81 81 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 41 41 41 76 76 76 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 38 38 38 69 69 69 F F F 10 200-219 44 44 44 80 80 80 F F F 11 220-239 43 43 43 79 79 79 F F F 12 240-259 41 41 41 76 76 76 F F F 13 260-279 44 44 44 82 82 82 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 44 44 44 80 80 80 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 44 44 44 80 80 80 F F F 19 380-399 44 44 44 82 82 82 T T T 20 400-419 44 44 44 80 80 80 F F F 21 420-439 44 44 44 82 82 82 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 38 38 38 69 69 69 T T T 24 480-499 38 38 38 69 69 69 F F F 25 500-519 44 44 44 82 82 82 F F F 26 520-539 44 44 44 80 80 80 F T T <--- 27 540-559 42 42 42 78 78 78 F F F 28 560-579 43 43 43 79 79 79 F F F 29 580-599 42 42 42 78 78 78 T T T Max gen: 44, Max flush: 82 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 81 81 81 F F F 1 020-039 44 44 44 81 81 81 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 41 41 41 76 76 76 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 37 37 37 68 68 68 F F F 6 120-139 30 30 30 58 58 58 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 38 38 38 69 69 69 F F F 10 200-219 44 44 44 80 80 80 F F F 11 220-239 43 43 43 79 79 79 F F F 12 240-259 41 41 41 76 76 76 F F F 13 260-279 44 44 44 82 82 82 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 44 44 44 80 80 80 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 44 44 44 80 80 80 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 44 44 44 80 80 80 F F F 21 420-439 44 44 44 82 82 82 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 44 44 44 82 82 82 F F F 26 520-539 45 45 45 83 83 83 F F F 27 540-559 42 42 42 78 78 78 F F F 28 560-579 43 43 43 79 79 79 F F F 29 580-599 45 45 45 83 83 83 F F F Max gen: 45, Max flush: 83 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.259794296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:49.302591959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.303230388Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.335600144Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.336680487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.33671734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.336736217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.336750864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366} {"msg":"Crucible 369f2573-5136-4af3-9c24-e667b0536e49 has session id: 461d45e0-195e-4b29-af64-07614ea536a3","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.337265825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Upstairs opts: Upstairs UUID: 369f2573-5136-4af3-9c24-e667b0536e49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.337312297Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Crucible stats registered with UUID: 369f2573-5136-4af3-9c24-e667b0536e49","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.337334983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.36867016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366} {"msg":"369f2573-5136-4af3-9c24-e667b0536e49 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.368740737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.368772822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.368815784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.368851488Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.368884882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.368918337Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.36895272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.368979236Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.36901608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.369046884Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42743)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.369075609Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55414)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.369101075Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:44477)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.369123281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.369865842Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.369903395Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.370087413Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.370116868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.370206142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.370233138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.370548833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.370757666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371073622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371250911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371295773Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.37131355Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371326427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371340095Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371353703Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371670487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371691304Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[0]R flush_numbers[0..12]: [81, 81, 70, 76, 79, 68, 58, 75, 81, 69, 80, 79]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.37171623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[0]R generation[0..12]: [44, 44, 39, 41, 43, 37, 30, 40, 44, 38, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371732547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371747044Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[1]R flush_numbers[0..12]: [81, 81, 70, 76, 79, 68, 58, 75, 81, 69, 80, 79]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371776549Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[1]R generation[0..12]: [44, 44, 39, 41, 43, 37, 30, 40, 44, 38, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371790687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371804514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[2]R flush_numbers[0..12]: [81, 81, 70, 76, 79, 68, 58, 75, 81, 69, 80, 79]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371818912Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[2]R generation[0..12]: [44, 44, 39, 41, 43, 37, 30, 40, 44, 38, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371835369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371849487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371862904Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371876122Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371889209Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371902247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"":"downstairs","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371915465Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1"Wait for a query_work_queue command to finish before sending IO ,"pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.37194341Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"0","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371956598Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"1","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371969815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"client":"2","":"downstairs","session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371983033Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.371995711Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"369f2573-5136-4af3-9c24-e667b0536e49 is now active with session: 461d45e0-195e-4b29-af64-07614ea536a3","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.372008929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.372021097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366,"session_id":"461d45e0-195e-4b29-af64-07614ea536a3"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:49.372033444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 564 len 24576 data: 4 5 3 4 4 4 02/30 Write block 65 len 20480 data: 3 5 5 5 5 03/30 Flush 04/30 Write block 549 len 16384 data: 9 10 9 9 05/30 Read block 169 len 16384 06/30 Read block 516 len 4096 07/30 Write block 369 len 28672 data: 4 5 6 7 8 8 7 08/30 Write block 190 len 12288 data: 6 4 4 09/30 Write block 114 len 28672 data: 4 3 3 3 5 4 3 10/30 Read block 216 len 8192 11/30 Read block 73 len 24576 12/30 Write block 364 len 36864 data: 9 7 7 6 4 5 6 7 8 13/30 Read block 35 len 28672 14/30 Read block 572 len 36864 15/30 Write block 278 len 36864 data: 4 4 4 4 5 4 2 3 4 16/30 Flush 17/30 Write block 237 len 28672 data: 6 6 6 5 4 3 3 18/30 Flush 19/30 Write block 502 len 4096 data: 3 20/30 Write block 124 len 16384 data: 5 5 5 4 21/30 Read block 211 len 32768 22/30 Read block 274 len 28672 23/30 Write block 554 len 8192 data: 6 5 24/30 Write block 561 len 40960 data: 7 6 4 5 6 4 5 5 5 3 25/30 Flush 26/30 Write block 188 len 4096 data: 6 27/30 Read block 447 len 36864 28/30 Read block 585 len 36864 29/30 Write block 21 len 4096 data: 6 30/30 Read block 120 len 16384 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 1 Sent Done Sent false 30 Acked 1029 Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 3 5 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:49.400237846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 81 81 81 F F F 1 020-039 44 44 44 81 81 81 T T T 2 040-059 39 39 39 70 70 70 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 46 46 46 85 85 85 F F F 6 120-139 46 46 46 87 87 87 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 46 46 46 85 85 85 T T T 10 200-219 44 44 44 80 80 80 F F F 11 220-239 46 46 46 86 86 86 F F F 12 240-259 46 46 46 86 86 86 F F F 13 260-279 46 46 46 85 85 85 F F F 14 280-299 46 46 46 85 85 85 F F F 15 300-319 44 44 44 80 80 80 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 44 44 44 80 80 80 F F F 21 420-439 44 44 44 82 82 82 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 46 46 46 87 87 87 F F F 26 520-539 45 45 45 83 83 83 F F F 27 540-559 46 46 46 87 87 87 F F F 28 560-579 46 46 46 87 87 87 F F F 29 580-599 45 45 45 83 83 83 F F F Max gen: 46, Max flush: 87 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 81 81 81 F F F 1 020-039 47 47 47 88 88 88 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 46 46 46 85 85 85 F F F 6 120-139 46 46 46 87 87 87 F F F 7 140-159 40 40 40 75 75 75 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 47 47 47 88 88 88 F F F 10 200-219 44 44 44 80 80 80 F F F 11 220-239 46 46 46 86 86 86 F F F 12 240-259 46 46 46 86 86 86 F F F 13 260-279 46 46 46 85 85 85 F F F 14 280-299 46 46 46 85 85 85 F F F 15 300-319 44 44 44 80 80 80 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 34 34 34 63 63 63 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 44 44 44 80 80 80 F F F 21 420-439 44 44 44 82 82 82 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 46 46 46 87 87 87 F F F 26 520-539 45 45 45 83 83 83 F F F 27 540-559 46 46 46 87 87 87 F F F 28 560-579 46 46 46 87 87 87 F F F 29 580-599 45 45 45 83 83 83 F F F Max gen: 47, Max flush: 88 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.717874801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:53.761061256Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.761625678Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.793547302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.794452865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.794491248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.794505495Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.794519843Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374} {"msg":"Crucible e90e8a59-f66d-468e-a488-0a54b5aefeda has session id: 41b4e2a7-65e6-4a39-8deb-80413a7247b5","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.795030504Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Upstairs opts: Upstairs UUID: e90e8a59-f66d-468e-a488-0a54b5aefeda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.795076236Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Crucible stats registered with UUID: e90e8a59-f66d-468e-a488-0a54b5aefeda","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.795102472Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826030259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374} {"msg":"e90e8a59-f66d-468e-a488-0a54b5aefeda active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826076861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826093008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826114585Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826128702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.82614242Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826156428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826170425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826183793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.82619867Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826222496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58238)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826356613Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:64322)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826408554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:49148)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.826676707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.827355739Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.827397321Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.827428996Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.82746648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.827492075Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.82752008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828091731Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828241675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828302994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828679249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828734859Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828752006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828904109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828947832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.828976947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829003302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829028628Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[0]R flush_numbers[0..12]: [81, 88, 70, 84, 79, 85, 87, 75, 81, 88, 80, 86]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829087287Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[0]R generation[0..12]: [44, 47, 39, 46, 43, 46, 46, 40, 44, 47, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829114943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.82913189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[1]R flush_numbers[0..12]: [81, 88, 70, 84, 79, 85, 87, 75, 81, 88, 80, 86]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829146337Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[1]R generation[0..12]: [44, 47, 39, 46, 43, 46, 46, 40, 44, 47, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829160135Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829174142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[2]R flush_numbers[0..12]: [81, 88, 70, 84, 79, 85, 87, 75, 81, 88, 80, 86]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829188Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[2]R generation[0..12]: [44, 47, 39, 46, 43, 46, 46, 40, 44, 47, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829201718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829215705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829228723Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.82924188Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829258008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829271555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"":"downstairs","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:11:53.829285373Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829306129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"0","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829319847Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"1","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829333505Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"client":"2","":"downstairs","session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829347022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.82935926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"e90e8a59-f66d-468e-a488-0a54b5aefeda is now active with session: 41b4e2a7-65e6-4a39-8deb-80413a7247b5","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829372218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829384186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374,"session_id":"41b4e2a7-65e6-4a39-8deb-80413a7247b5"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:53.829396474Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 12 len 8192 data: 9 10 02/30 Read block 417 len 24576 03/30 Write block 51 len 16384 data: 6 6 5 6 04/30 Write block 539 len 20480 data: 7 8 7 6 5 05/30 Read block 447 len 4096 06/30 Read block 550 len 4096 07/30 Read block 283 len 24576 08/30 Read block 433 len 4096 09/30 Write block 145 len 8192 data: 4 5 10/30 Read block 491 len 8192 11/30 Read block 441 len 12288 12/30 Flush 13/30 Write block 82 len 28672 data: 4 6 7 5 5 5 4 14/30 Read block 293 len 16384 15/30 Write block 335 len 40960 data: 5 5 5 6 6 6 5 4 3 4 16/30 Read block 27 len 40960 17/30 Flush 18/30 Write block 106 len 20480 data: 7 6 6 5 5 19/30 Write block 76 len 20480 data: 6 4 4 6 5 20/30 Read block 390 len 32768 21/30 Read block 27 len 4096 22/30 Write block 507 len 4096 data: 4 23/30 Write block 253 len 16384 data: 4 6 6 8 24/30 Write block 274 len 12288 data: 3 3 4 25/30 Read block 434 len 24576 26/30 Read block 144 len 36864 27/30 Write block 63 len 40960 data: 4 3 4 6 6 6 6 7 8 9 28/30 Read block 480 len 24576 29/30 Write block 131 len 8192 data: 2 2 30/30 Write block 350 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 8 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 2 Sent Sent Sent false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:53.852512981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 47 47 47 88 88 88 F F F 2 040-059 48 48 48 89 89 89 F F F 3 060-079 46 46 46 84 84 84 T T T 4 080-099 48 48 48 90 90 90 T T T 5 100-119 46 46 46 85 85 85 T T T 6 120-139 46 46 46 87 87 87 T T T 7 140-159 48 48 48 89 89 89 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 47 47 47 88 88 88 F F F 10 200-219 44 44 44 80 80 80 F F F 11 220-239 46 46 46 86 86 86 F F F 12 240-259 46 46 46 86 86 86 T T T 13 260-279 46 46 46 85 85 85 T T T 14 280-299 46 46 46 85 85 85 F F F 15 300-319 44 44 44 80 80 80 F F F 16 320-339 48 48 48 90 90 90 F F F 17 340-359 48 48 48 90 90 90 T T T 18 360-379 46 46 46 85 85 85 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 44 44 44 80 80 80 F F F 21 420-439 44 44 44 82 82 82 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 46 46 46 87 87 87 T T T 26 520-539 48 48 48 89 89 89 F F F 27 540-559 48 48 48 89 89 89 F F F 28 560-579 46 46 46 87 87 87 F F F 29 580-599 45 45 45 83 83 83 F F F Max gen: 48, Max flush: 90 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 47 47 47 88 88 88 F F F 2 040-059 48 48 48 89 89 89 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 49 49 49 91 91 91 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 49 49 49 91 91 91 F F F 7 140-159 48 48 48 89 89 89 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 47 47 47 88 88 88 F F F 10 200-219 44 44 44 80 80 80 F F F 11 220-239 46 46 46 86 86 86 F F F 12 240-259 49 49 49 91 91 91 F F F 13 260-279 49 49 49 91 91 91 F F F 14 280-299 46 46 46 85 85 85 F F F 15 300-319 44 44 44 80 80 80 F F F 16 320-339 48 48 48 90 90 90 F F F 17 340-359 49 49 49 91 91 91 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 44 44 44 80 80 80 F F F 21 420-439 44 44 44 82 82 82 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 49 49 49 91 91 91 F F F 26 520-539 48 48 48 89 89 89 F F F 27 540-559 48 48 48 89 89 89 F F F 28 560-579 46 46 46 87 87 87 F F F 29 580-599 45 45 45 83 83 83 F F F Max gen: 49, Max flush: 91 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.252670295Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.296875822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.297780805Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.297817168Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.297832106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.297846423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382} {"msg":"Crucible fd607b0e-5f0f-46a0-b21a-8cae1c2f09a8 has session id: 150f62be-aa00-46aa-b756-8aa6f8cdff96","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.29826474Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Upstairs opts: Upstairs UUID: fd607b0e-5f0f-46a0-b21a-8cae1c2f09a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.29832396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Crucible stats registered with UUID: fd607b0e-5f0f-46a0-b21a-8cae1c2f09a8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.298347846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.329941208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382} {"msg":"fd607b0e-5f0f-46a0-b21a-8cae1c2f09a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.329999938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330019474Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330048739Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330066196Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330083383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.33009966Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330118117Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330139993Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330235827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330262652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_connection connected from Ok(127.0.0.1:41395)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330394849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_connection connected from Ok(127.0.0.1:37612)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330441961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_connection connected from Ok(127.0.0.1:41613)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.330574968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.331363261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.331393875Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.331414562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.331429349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.331442767Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.331456314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332119249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332143755Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.33217109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332798431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332827426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332843083Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332855951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332869718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332883226Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332895704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332909182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[0]R flush_numbers[0..12]: [89, 88, 89, 91, 91, 91, 91, 89, 81, 88, 80, 86]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332926519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[0]R generation[0..12]: [48, 47, 48, 49, 49, 49, 49, 48, 44, 47, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332940896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332954864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[1]R flush_numbers[0..12]: [89, 88, 89, 91, 91, 91, 91, 89, 81, 88, 80, 86]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332968491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[1]R generation[0..12]: [48, 47, 48, 49, 49, 49, 49, 48, 44, 47, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332981949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.332995707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[2]R flush_numbers[0..12]: [89, 88, 89, 91, 91, 91, 91, 89, 81, 88, 80, 86]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333009034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[2]R generation[0..12]: [48, 47, 48, 49, 49, 49, 49, 48, 44, 47, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333022522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.33303632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333049157Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333062235Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333074933Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333104288Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"":"downstairs","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333118115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:11:58.333130193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"0","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333164907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"1","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333191142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"client":"2","":"downstairs","session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333218698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333242604Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"fd607b0e-5f0f-46a0-b21a-8cae1c2f09a8 is now active with session: 150f62be-aa00-46aa-b756-8aa6f8cdff96","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333269489Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.333294715Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382,"session_id":"150f62be-aa00-46aa-b756-8aa6f8cdff96"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:11:58.33331983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 288 len 28672 data: 4 4 2 2 2 2 2 02/30 Write block 306 len 28672 data: 7 6 5 5 6 6 5 03/30 Read block 24 len 24576 04/30 Read block 279 len 4096 05/30 Read block 279 len 20480 06/30 Write block 190 len 8192 data: 7 5 07/30 Read block 247 len 40960 08/30 Write block 163 len 40960 data: 4 3 3 5 5 7 5 4 5 5 09/30 Write block 408 len 32768 data: 4 3 4 4 5 6 6 7 10/30 Read block 354 len 32768 11/30 Read block 200 len 24576 12/30 Write block 261 len 12288 data: 5 4 5 13/30 Write block 280 len 32768 data: 5 5 6 5 3 4 5 4 14/30 Read block 466 len 16384 15/30 Read block 213 len 28672 16/30 Read block 376 len 36864 17/30 Write block 434 len 8192 data: 4 4 18/30 Write block 254 len 36864 data: 7 7 9 7 6 6 7 6 5 19/30 Flush 20/30 Write block 306 len 28672 data: 8 7 6 6 7 7 6 21/30 Write block 225 len 12288 data: 4 5 5 22/30 Read block 436 len 16384 23/30 Write block 592 len 4096 data: 7 24/30 Write block 579 len 12288 data: 6 6 6 25/30 Read block 429 len 36864 26/30 Read block 130 len 40960 27/30 Read block 552 len 40960 28/30 Read block 442 len 24576 29/30 Write block 542 len 12288 data: 7 6 6 30/30 Read block 66 len 24576 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 7 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 3 Done Sent Sent false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 11 9 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:11:58.357118249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 47 47 47 88 88 88 F F F 2 040-059 48 48 48 89 89 89 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 49 49 49 91 91 91 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 49 49 49 91 91 91 F F F 7 140-159 48 48 48 89 89 89 F F F 8 160-179 50 50 50 92 92 92 F F F 9 180-199 50 50 50 92 92 92 F F F 10 200-219 44 44 44 80 80 80 F F F 11 220-239 46 46 46 86 86 86 T T T 12 240-259 50 50 50 92 92 92 F F F 13 260-279 50 50 50 92 92 92 F F F 14 280-299 50 50 50 92 92 92 F F F 15 300-319 50 50 50 92 92 92 T T T 16 320-339 48 48 48 90 90 90 F F F 17 340-359 49 49 49 91 91 91 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 50 50 50 92 92 92 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 49 49 49 91 91 91 F F F 26 520-539 48 48 48 89 89 89 F F F 27 540-559 48 48 48 89 89 89 T T T 28 560-579 46 46 46 87 87 87 T T T 29 580-599 45 45 45 83 83 83 T T T Max gen: 50, Max flush: 92 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 47 47 47 88 88 88 F F F 2 040-059 48 48 48 89 89 89 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 49 49 49 91 91 91 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 49 49 49 91 91 91 F F F 7 140-159 48 48 48 89 89 89 F F F 8 160-179 50 50 50 92 92 92 F F F 9 180-199 50 50 50 92 92 92 F F F 10 200-219 44 44 44 80 80 80 F F F 11 220-239 51 51 51 93 93 93 F F F 12 240-259 50 50 50 92 92 92 F F F 13 260-279 50 50 50 92 92 92 F F F 14 280-299 50 50 50 92 92 92 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 48 48 48 90 90 90 F F F 17 340-359 49 49 49 91 91 91 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 50 50 50 92 92 92 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 49 49 49 91 91 91 F F F 26 520-539 48 48 48 89 89 89 F F F 27 540-559 51 51 51 93 93 93 F F F 28 560-579 51 51 51 93 93 93 F F F 29 580-599 51 51 51 93 93 93 F F F Max gen: 51, Max flush: 93 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.678603134Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.72281735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.723661413Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.723699026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.723713584Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.723727931Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390} {"msg":"Crucible b75006e5-e54b-402a-8ce1-2cab4eb85e33 has session id: c8ea604e-6000-4d10-9c3f-1cdd835c48c9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.724292323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Upstairs opts: Upstairs UUID: b75006e5-e54b-402a-8ce1-2cab4eb85e33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.724345314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Crucible stats registered with UUID: b75006e5-e54b-402a-8ce1-2cab4eb85e33","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.724371459Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.75614585Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390} {"msg":"b75006e5-e54b-402a-8ce1-2cab4eb85e33 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756211659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.75626008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756306642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756354294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756392377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756428731Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756464934Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.75649273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756527393Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756565717Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41660)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33677)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756641793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64241)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.756680577Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.757427327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.757462781Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.757479858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.757494055Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.7576371Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.757681273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758177176Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758205771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758221129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758797968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758822404Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758836182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758848449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758861567Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758874475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758886583Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.75890012Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[0]R flush_numbers[0..12]: [89, 88, 89, 91, 91, 91, 91, 89, 92, 92, 80, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758913828Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[0]R generation[0..12]: [48, 47, 48, 49, 49, 49, 49, 48, 50, 50, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758927566Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758941623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[1]R flush_numbers[0..12]: [89, 88, 89, 91, 91, 91, 91, 89, 92, 92, 80, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.75895896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[1]R generation[0..12]: [48, 47, 48, 49, 49, 49, 49, 48, 50, 50, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758972618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758986385Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[2]R flush_numbers[0..12]: [89, 88, 89, 91, 91, 91, 91, 89, 92, 92, 80, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.758999813Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[2]R generation[0..12]: [48, 47, 48, 49, 49, 49, 49, 48, 50, 50, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759013211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759027018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759039686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759052754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759065632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759089068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"":"downstairs","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759102255Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759114183Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"0","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759127301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"1","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759140479Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"client":"2","":"downstairs","session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759153647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759168054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"b75006e5-e54b-402a-8ce1-2cab4eb85e33 is now active with session: c8ea604e-6000-4d10-9c3f-1cdd835c48c9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759180922Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:12:02.75919296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390,"session_id":"c8ea604e-6000-4d10-9c3f-1cdd835c48c9"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:02.759206427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 216 len 20480 data: 4 4 4 4 3 02/30 Read block 527 len 16384 03/30 Write block 276 len 8192 data: 5 4 04/30 Flush 05/30 Write block 180 len 4096 data: 4 06/30 Read block 35 len 20480 07/30 Write block 509 len 20480 data: 6 5 5 5 6 08/30 Read block 352 len 28672 09/30 Read block 84 len 40960 10/30 Write block 540 len 20480 data: 9 8 8 7 7 11/30 Write block 577 len 40960 data: 6 6 7 7 7 6 7 4 5 4 12/30 Flush 13/30 Write block 367 len 40960 data: 7 5 6 7 8 9 9 9 8 6 14/30 Read block 353 len 4096 15/30 Read block 304 len 36864 16/30 Write block 155 len 40960 data: 5 4 4 4 4 5 4 5 5 4 17/30 Read block 22 len 4096 18/30 Write block 174 len 8192 data: 4 3 19/30 Write block 29 len 36864 data: 3 4 5 4 5 6 4 4 4 20/30 Flush 21/30 Write block 592 len 4096 data: 8 22/30 Read block 255 len 12288 23/30 Read block 582 len 4096 24/30 Write block 593 len 24576 data: 7 6 5 4 3 3 25/30 Write block 327 len 36864 data: 5 5 4 4 4 4 4 5 6 26/30 Write block 143 len 16384 data: 5 4 5 6 27/30 Write block 270 len 40960 data: 3 3 3 4 4 4 6 5 5 5 28/30 Write block 127 len 24576 data: 5 4 4 4 3 3 29/30 Write block 289 len 4096 data: 5 30/30 Write block 470 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 10 Sent Sent Sent false 28 Acked 1027 Write 6 Sent Sent Sent false 29 Acked 1028 Write 1 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:02.783714712Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 52 52 52 96 96 96 F F F 2 040-059 48 48 48 89 89 89 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 49 49 49 91 91 91 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 49 49 49 91 91 91 T T T 7 140-159 52 52 52 96 96 96 T T T 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 52 52 52 94 94 94 F F F 12 240-259 50 50 50 92 92 92 F F F 13 260-279 52 52 52 94 94 94 T T T 14 280-299 50 50 50 92 92 92 T T T 15 300-319 51 51 51 93 93 93 F F F 16 320-339 48 48 48 90 90 90 T T T 17 340-359 49 49 49 91 91 91 F F F 18 360-379 52 52 52 96 96 96 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 50 50 50 92 92 92 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 45 45 45 83 83 83 T T T 24 480-499 38 38 38 69 69 69 F F F 25 500-519 52 52 52 95 95 95 F F F 26 520-539 48 48 48 89 89 89 F F F 27 540-559 52 52 52 95 95 95 F F F 28 560-579 52 52 52 95 95 95 F F F 29 580-599 52 52 52 95 95 95 T T T Max gen: 52, Max flush: 96 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 52 52 52 96 96 96 F F F 2 040-059 48 48 48 89 89 89 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 49 49 49 91 91 91 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 53 53 53 97 97 97 F F F 7 140-159 53 53 53 97 97 97 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 52 52 52 94 94 94 F F F 12 240-259 50 50 50 92 92 92 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 53 53 53 97 97 97 F F F 17 340-359 49 49 49 91 91 91 F F F 18 360-379 52 52 52 96 96 96 F F F 19 380-399 45 45 45 83 83 83 F F F 20 400-419 50 50 50 92 92 92 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 44 44 44 80 80 80 F F F 23 460-479 53 53 53 97 97 97 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 52 52 52 95 95 95 F F F 26 520-539 48 48 48 89 89 89 F F F 27 540-559 52 52 52 95 95 95 F F F 28 560-579 52 52 52 95 95 95 F F F 29 580-599 53 53 53 97 97 97 F F F Max gen: 53, Max flush: 97 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.133417627Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.178197465Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.179239223Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.179307732Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.179335477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.179364391Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398} {"msg":"Crucible 4d5a6c85-c892-4c4f-9abb-85c1ce863ea0 has session id: d1ca0a86-66ab-4805-925c-d850b723f261","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.17994745Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Upstairs opts: Upstairs UUID: 4d5a6c85-c892-4c4f-9abb-85c1ce863ea0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.180024097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Crucible stats registered with UUID: 4d5a6c85-c892-4c4f-9abb-85c1ce863ea0","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.180052192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.21161031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398} {"msg":"4d5a6c85-c892-4c4f-9abb-85c1ce863ea0 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.21166757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211686987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211719081Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211737388Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211754535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211771322Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211788199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211810425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211828132Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.211845109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_connection connected from Ok(127.0.0.1:57248)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.212038416Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_connection connected from Ok(127.0.0.1:64039)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.212087517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_connection connected from Ok(127.0.0.1:63269)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.212249029Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213066997Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213101881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213128586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213155511Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213183686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213203503Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213711985Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.2138516Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.213892123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214283065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.21431189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214327737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214400135Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214419451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214433069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214505226Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214524413Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[0]R flush_numbers[0..12]: [89, 96, 89, 91, 91, 91, 97, 97, 96, 95, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214550009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[0]R generation[0..12]: [48, 52, 48, 49, 49, 49, 53, 53, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214564426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214578684Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[1]R flush_numbers[0..12]: [89, 96, 89, 91, 91, 91, 97, 97, 96, 95, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214592411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[1]R generation[0..12]: [48, 52, 48, 49, 49, 49, 53, 53, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214605959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214621636Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[2]R flush_numbers[0..12]: [89, 96, 89, 91, 91, 91, 97, 97, 96, 95, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214635464Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[2]R generation[0..12]: [48, 52, 48, 49, 49, 49, 53, 53, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214652451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214666958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214679856Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214692894Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214705672Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214718529Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"":"downstairs","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214731707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214755483Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"0","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.21476877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"1","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214782018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"client":"2","":"downstairs","session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214795376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214807704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"4d5a6c85-c892-4c4f-9abb-85c1ce863ea0 is now active with session: d1ca0a86-66ab-4805-925c-d850b723f261","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214820561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214832439Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398,"session_id":"d1ca0a86-66ab-4805-925c-d850b723f261"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:07.214844737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 404 len 40960 data: 4 4 5 4 5 4 5 5 6 7 02/30 Read block 119 len 24576 03/30 Read block 79 len 16384 04/30 Read block 418 len 20480 05/30 Read block 551 len 32768 06/30 Flush 07/30 Write block 79 len 8192 data: 7 6 08/30 Read block 133 len 28672 09/30 Read block 456 len 28672 10/30 Write block 32 len 8192 data: 5 6 11/30 Read block 11 len 40960 12/30 Read block 284 len 8192 13/30 Read block 200 len 4096 14/30 Read block 267 len 24576 15/30 Write block 570 len 20480 data: 4 4 5 5 6 16/30 Read block 476 len 28672 17/30 Read block 497 len 24576 18/30 Write block 145 len 16384 data: 6 7 5 5 19/30 Read block 4 len 20480 20/30 Flush 21/30 Write block 456 len 36864 data: 4 4 4 3 3 2 4 5 6 22/30 Write block 241 len 8192 data: 5 4 23/30 Read block 102 len 4096 24/30 Write block 209 len 12288 data: 3 3 3 25/30 Read block 74 len 12288 26/30 Write block 57 len 24576 data: 5 5 3 3 3 4 27/30 Write block 44 len 28672 data: 5 6 6 7 7 6 5 28/30 Read block 30 len 12288 29/30 Read block 541 len 36864 30/30 Write block 395 len 24576 data: 3 3 4 4 4 5 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:07.23810423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 54 54 54 99 99 99 F F F 2 040-059 48 48 48 89 89 89 T T T 3 060-079 54 54 54 99 99 99 T T T 4 080-099 54 54 54 99 99 99 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 53 53 53 97 97 97 F F F 7 140-159 54 54 54 99 99 99 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 52 52 52 94 94 94 T T T 11 220-239 52 52 52 94 94 94 F F F 12 240-259 50 50 50 92 92 92 T T T 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 53 53 53 97 97 97 F F F 17 340-359 49 49 49 91 91 91 F F F 18 360-379 52 52 52 96 96 96 F F F 19 380-399 45 45 45 83 83 83 T T T 20 400-419 54 54 54 98 98 98 T T T 21 420-439 50 50 50 92 92 92 F F F 22 440-459 44 44 44 80 80 80 T T T 23 460-479 53 53 53 97 97 97 T T T 24 480-499 38 38 38 69 69 69 F F F 25 500-519 52 52 52 95 95 95 F F F 26 520-539 48 48 48 89 89 89 F F F 27 540-559 52 52 52 95 95 95 F F F 28 560-579 54 54 54 99 99 99 F F F 29 580-599 53 53 53 97 97 97 F F F Max gen: 54, Max flush: 99 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 54 54 54 99 99 99 F F F 2 040-059 55 55 55 100 100 100 F F F 3 060-079 55 55 55 100 100 100 F F F 4 080-099 54 54 54 99 99 99 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 53 53 53 97 97 97 F F F 7 140-159 54 54 54 99 99 99 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 55 55 55 100 100 100 F F F 11 220-239 52 52 52 94 94 94 F F F 12 240-259 55 55 55 100 100 100 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 53 53 53 97 97 97 F F F 17 340-359 49 49 49 91 91 91 F F F 18 360-379 52 52 52 96 96 96 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 55 55 55 100 100 100 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 55 55 55 100 100 100 F F F 23 460-479 55 55 55 100 100 100 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 52 52 52 95 95 95 F F F 26 520-539 48 48 48 89 89 89 F F F 27 540-559 52 52 52 95 95 95 F F F 28 560-579 54 54 54 99 99 99 F F F 29 580-599 53 53 53 97 97 97 F F F Max gen: 55, Max flush: 100 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.603011339Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:11.646612561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.647208907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.679538461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.68046618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.680502574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.680516761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.680535178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406} {"msg":"Crucible 0c7caf99-3f07-4802-8c04-476545ab88ff has session id: a24f0c91-05a4-431e-aa2b-5fe0ab5a025a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.681052468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Upstairs opts: Upstairs UUID: 0c7caf99-3f07-4802-8c04-476545ab88ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.681106728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Crucible stats registered with UUID: 0c7caf99-3f07-4802-8c04-476545ab88ff","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.681129185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712041525Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406} {"msg":"0c7caf99-3f07-4802-8c04-476545ab88ff active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712092396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712110183Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712139458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712165004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712198578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712231182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712264236Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.71229974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712335564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712372968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50763)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712405382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33411)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712440586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33264)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.712683194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.713354537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.713396549Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.713427204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.71344823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.713470386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.713484484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714053965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714103286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714384787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714740825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.71477078Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714812353Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714825851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714839608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714853196Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.714990612Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715010858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[0]R flush_numbers[0..12]: [89, 99, 100, 100, 99, 91, 97, 99, 96, 95, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715068488Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[0]R generation[0..12]: [48, 54, 55, 55, 54, 49, 53, 54, 52, 52, 55, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715089045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715103692Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[1]R flush_numbers[0..12]: [89, 99, 100, 100, 99, 91, 97, 99, 96, 95, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.71511713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[1]R generation[0..12]: [48, 54, 55, 55, 54, 49, 53, 54, 52, 52, 55, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715130737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715144555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[2]R flush_numbers[0..12]: [89, 99, 100, 100, 99, 91, 97, 99, 96, 95, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715158043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[2]R generation[0..12]: [48, 54, 55, 55, 54, 49, 53, 54, 52, 52, 55, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.71517163Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715185448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715201865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715215333Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.71522812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715241138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"":"downstairs","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715253986Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"Wait for a query_work_queue command to finish before sending IO a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715277162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"0","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.71529129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"1","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715304277Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"client":"2","":"downstairs","session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715317375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715329653Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"0c7caf99-3f07-4802-8c04-476545ab88ff is now active with session: a24f0c91-05a4-431e-aa2b-5fe0ab5a025a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715342651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715354669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406,"session_id":"a24f0c91-05a4-431e-aa2b-5fe0ab5a025a"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:11.715366936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 248 len 40960 data: 3 3 4 4 3 5 8 8 10 8 02/30 Read block 535 len 32768 03/30 Read block 368 len 40960 04/30 Read block 343 len 8192 05/30 Write block 579 len 36864 data: 8 8 8 7 8 5 6 5 4 06/30 Write block 359 len 20480 data: 4 4 5 8 8 07/30 Read block 590 len 40960 08/30 Read block 332 len 4096 09/30 Read block 133 len 4096 10/30 Read block 527 len 20480 11/30 Read block 170 len 12288 12/30 Write block 512 len 40960 data: 6 7 6 9 12 11 8 7 7 8 13/30 Read block 449 len 36864 14/30 Write block 565 len 12288 data: 7 5 6 15/30 Read block 146 len 12288 16/30 Write block 575 len 8192 data: 7 6 17/30 Write block 79 len 8192 data: 8 7 18/30 Read block 375 len 24576 19/30 Write block 44 len 4096 data: 6 20/30 Read block 362 len 8192 21/30 Read block 27 len 12288 22/30 Read block 332 len 40960 23/30 Read block 133 len 36864 24/30 Write block 589 len 12288 data: 5 6 7 25/30 Read block 375 len 24576 26/30 Write block 24 len 16384 data: 4 4 4 3 27/30 Read block 256 len 4096 28/30 Read block 288 len 16384 29/30 Write block 206 len 36864 data: 4 4 3 4 4 4 2 4 4 30/30 Write block 200 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 10 Done Done Done false 2 Acked 1001 Read 8 Done Done Done false 3 Acked 1002 Read 10 Done Done Done false 4 Acked 1003 Read 2 Done Done Done false 5 Acked 1004 Write 9 Done Done Done false 6 Acked 1005 Write 5 Done Done Done false 7 Acked 1006 Read 10 Done Done Done false 8 Acked 1007 Read 1 Done Done Done false 9 Acked 1008 Read 1 Done Done Done false 10 Acked 1009 Read 5 Done Done Done false 11 Acked 1010 Read 3 Done Done Done false 12 Acked 1011 Write 10 Done Done Done false 13 Acked 1012 Read 9 Done Done Done false 14 Acked 1013 Write 3 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Write 2 Done Done Done false 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Read 6 Done Done Done false 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:11.736036179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 54 54 54 99 99 99 T T T 2 040-059 55 55 55 100 100 100 T T T 3 060-079 55 55 55 100 100 100 T T T 4 080-099 54 54 54 99 99 99 T T T 5 100-119 49 49 49 91 91 91 F F F 6 120-139 53 53 53 97 97 97 F F F 7 140-159 54 54 54 99 99 99 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 55 55 55 100 100 100 T T T 11 220-239 52 52 52 94 94 94 F F F 12 240-259 55 55 55 100 100 100 T T T 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 53 53 53 97 97 97 F F F 17 340-359 49 49 49 91 91 91 T T T 18 360-379 52 52 52 96 96 96 T T T 19 380-399 55 55 55 100 100 100 F F F 20 400-419 55 55 55 100 100 100 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 55 55 55 100 100 100 F F F 23 460-479 55 55 55 100 100 100 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 52 52 52 95 95 95 T T T 26 520-539 48 48 48 89 89 89 T T T 27 540-559 52 52 52 95 95 95 F F F 28 560-579 54 54 54 99 99 99 T T T 29 580-599 53 53 53 97 97 97 T T T Max gen: 55, Max flush: 100 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 89 89 89 F F F 1 020-039 56 56 56 101 101 101 F F F 2 040-059 56 56 56 101 101 101 F F F 3 060-079 56 56 56 101 101 101 F F F 4 080-099 56 56 56 101 101 101 F F F 5 100-119 49 49 49 91 91 91 F F F 6 120-139 53 53 53 97 97 97 F F F 7 140-159 54 54 54 99 99 99 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 56 56 56 101 101 101 F F F 11 220-239 52 52 52 94 94 94 F F F 12 240-259 56 56 56 101 101 101 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 53 53 53 97 97 97 F F F 17 340-359 56 56 56 101 101 101 F F F 18 360-379 56 56 56 101 101 101 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 55 55 55 100 100 100 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 55 55 55 100 100 100 F F F 23 460-479 55 55 55 100 100 100 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 56 56 56 101 101 101 F F F 26 520-539 56 56 56 101 101 101 F F F 27 540-559 52 52 52 95 95 95 F F F 28 560-579 56 56 56 101 101 101 F F F 29 580-599 56 56 56 101 101 101 F F F Max gen: 56, Max flush: 101 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.187238501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.232512203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.233356926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.233396259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.233415096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.233430243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414} {"msg":"Crucible 95598896-a732-4b9e-b1cc-4bb978dbb35e has session id: 8567a20d-be2b-48a4-9375-442480dc02d6","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.233888074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Upstairs opts: Upstairs UUID: 95598896-a732-4b9e-b1cc-4bb978dbb35e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.233936215Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Crucible stats registered with UUID: 95598896-a732-4b9e-b1cc-4bb978dbb35e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.233961671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.2647684Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414} {"msg":"95598896-a732-4b9e-b1cc-4bb978dbb35e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26482334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.264844606Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.264874571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.264904836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26493791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.264968925Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.265002829Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.265036663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.265072517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26511341Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60641)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.265147684Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53038)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.265183348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:50958)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.265469698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.266152079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.266181214Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26620357Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.266229575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.266258121Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.266284756Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26694823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.266978705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.266996892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267591888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267620153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26763864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267652008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267666035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267679483Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267691521Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267704869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[0]R flush_numbers[0..12]: [89, 101, 101, 101, 101, 91, 97, 99, 96, 95, 101, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267718576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[0]R generation[0..12]: [48, 56, 56, 56, 56, 49, 53, 54, 52, 52, 56, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267732434Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267746391Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[1]R flush_numbers[0..12]: [89, 101, 101, 101, 101, 91, 97, 99, 96, 95, 101, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267759939Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[1]R generation[0..12]: [48, 56, 56, 56, 56, 49, 53, 54, 52, 52, 56, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267773197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267786884Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[2]R flush_numbers[0..12]: [89, 101, 101, 101, 101, 91, 97, 99, 96, 95, 101, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267800402Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[2]R generation[0..12]: [48, 56, 56, 56, 56, 49, 53, 54, 52, 52, 56, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26781403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267828247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Max found gen is 57","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267843465Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Generation requested: 58 >= found:57","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267857022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:12:16.26787002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267891956Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"":"downstairs","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267905254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267917092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"0","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.26793015Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"1","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267943117Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"client":"2","":"downstairs","session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267956415Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267968593Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"95598896-a732-4b9e-b1cc-4bb978dbb35e is now active with session: 8567a20d-be2b-48a4-9375-442480dc02d6","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-09-27T17:12:16.267981301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.267997458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414,"session_id":"8567a20d-be2b-48a4-9375-442480dc02d6"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:16.268009745Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 8192 data: 9 6 02/30 Write block 483 len 40960 data: 3 3 3 3 3 3 3 2 2 2 03/30 Flush 04/30 Write block 471 len 24576 data: 4 4 3 4 5 4 05/30 Read block 262 len 40960 06/30 Write block 474 len 32768 data: 5 6 5 4 3 3 3 2 07/30 Write block 23 len 20480 data: 4 5 5 5 4 08/30 Write block 335 len 40960 data: 7 6 6 7 7 7 6 5 4 5 09/30 Write block 41 len 40960 data: 6 6 6 7 7 7 8 8 7 6 10/30 Write block 99 len 4096 data: 4 11/30 Read block 91 len 16384 12/30 Read block 283 len 8192 13/30 Flush 14/30 Write block 131 len 36864 data: 4 4 3 5 4 5 6 5 6 15/30 Read block 295 len 40960 16/30 Write block 307 len 28672 data: 8 7 7 8 8 7 5 17/30 Write block 572 len 4096 data: 6 18/30 Write block 207 len 32768 data: 5 4 5 5 5 3 5 5 19/30 Read block 410 len 12288 20/30 Read block 217 len 36864 21/30 Write block 224 len 32768 data: 4 5 6 6 6 6 6 5 22/30 Write block 507 len 24576 data: 5 6 7 6 6 7 23/30 Read block 534 len 8192 24/30 Read block 571 len 28672 25/30 Read block 336 len 16384 26/30 Write block 116 len 36864 data: 4 4 6 5 4 5 6 6 6 27/30 Write block 454 len 32768 data: 5 4 5 5 5 4 4 3 28/30 Write block 445 len 8192 data: 3 6 29/30 Read block 245 len 40960 30/30 Write block 475 len 8192 data: 7 6 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 9 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Write 7 Done Done Done false 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 3 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:16.292034835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 102 102 102 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 58 58 58 103 103 103 F F F 3 060-079 56 56 56 101 101 101 F F F 4 080-099 58 58 58 103 103 103 F F F 5 100-119 49 49 49 91 91 91 T T T 6 120-139 53 53 53 97 97 97 T T T 7 140-159 54 54 54 99 99 99 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 56 56 56 101 101 101 T T T 11 220-239 52 52 52 94 94 94 T T T 12 240-259 56 56 56 101 101 101 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 F F F 15 300-319 51 51 51 93 93 93 T T T 16 320-339 58 58 58 103 103 103 F F F 17 340-359 58 58 58 103 103 103 F F F 18 360-379 56 56 56 101 101 101 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 55 55 55 100 100 100 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 55 55 55 100 100 100 T T T 23 460-479 58 58 58 103 103 103 T T T 24 480-499 58 58 58 103 103 103 F F F 25 500-519 56 56 56 101 101 101 T T T 26 520-539 56 56 56 101 101 101 F F F 27 540-559 52 52 52 95 95 95 F F F 28 560-579 56 56 56 101 101 101 T T T 29 580-599 56 56 56 101 101 101 F F F Max gen: 58, Max flush: 103 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 102 102 102 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 58 58 58 103 103 103 F F F 3 060-079 56 56 56 101 101 101 F F F 4 080-099 58 58 58 103 103 103 F F F 5 100-119 59 59 59 104 104 104 F F F 6 120-139 59 59 59 104 104 104 F F F 7 140-159 54 54 54 99 99 99 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 F F F 10 200-219 59 59 59 104 104 104 F F F 11 220-239 59 59 59 104 104 104 F F F 12 240-259 56 56 56 101 101 101 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 F F F 15 300-319 59 59 59 104 104 104 F F F 16 320-339 58 58 58 103 103 103 F F F 17 340-359 58 58 58 103 103 103 F F F 18 360-379 56 56 56 101 101 101 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 55 55 55 100 100 100 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 59 59 59 104 104 104 F F F 23 460-479 59 59 59 104 104 104 F F F 24 480-499 58 58 58 103 103 103 F F F 25 500-519 59 59 59 104 104 104 F F F 26 520-539 56 56 56 101 101 101 F F F 27 540-559 52 52 52 95 95 95 F F F 28 560-579 59 59 59 104 104 104 F F F 29 580-599 56 56 56 101 101 101 F F F Max gen: 59, Max flush: 104 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.674256481Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.718868527Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.719758203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.719798205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.719813113Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.71982822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422} {"msg":"Crucible 410c8301-9a20-438f-817b-299149d59d97 has session id: 94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.720298398Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Upstairs opts: Upstairs UUID: 410c8301-9a20-438f-817b-299149d59d97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.720343211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Crucible stats registered with UUID: 410c8301-9a20-438f-817b-299149d59d97","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.720367986Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752200657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422} {"msg":"410c8301-9a20-438f-817b-299149d59d97 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752255057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752274514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752304389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752322376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752340022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.75235692Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752374287Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752390884Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752409801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752441115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37501)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752903524Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36644)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.752973502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39643)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.753000867Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.753544603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.753571578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.753586666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.753600253Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.753744188Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.753817645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.754329096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.754360791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.754452575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.754971035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755011878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755040403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755070997Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755098243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755120349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755140155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755162771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[0]R flush_numbers[0..12]: [102, 103, 103, 101, 103, 104, 104, 99, 96, 95, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755188577Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[0]R generation[0..12]: [58, 58, 58, 56, 58, 59, 59, 54, 52, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755212723Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755235779Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[1]R flush_numbers[0..12]: [102, 103, 103, 101, 103, 104, 104, 99, 96, 95, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755258685Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[1]R generation[0..12]: [58, 58, 58, 56, 58, 59, 59, 54, 52, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.75528247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755308766Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[2]R flush_numbers[0..12]: [102, 103, 103, 101, 103, 104, 104, 99, 96, 95, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755335381Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[2]R generation[0..12]: [58, 58, 58, 56, 58, 59, 59, 54, 52, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755360047Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755385903Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:12:20.755417377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.75545755Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755485755Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755511791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"":"downstairs","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755539106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755559932Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"0","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.75557448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"1","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755587867Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"client":"2","":"downstairs","session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755601155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755613433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 410c8301-9a20-438f-817b-299149d59d97 is now active with session: 94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.75562908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755641168Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422,"session_id":"94ef92ba-e3f3-49cb-9fb3-ee8a9cdb57b8"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:20.755653476Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 579 len 20480 data: 9 9 9 8 9 02/30 Read block 225 len 4096 03/30 Write block 511 len 8192 data: 7 8 04/30 Flush 05/30 Write block 138 len 12288 data: 6 7 5 06/30 Write block 196 len 36864 data: 6 5 5 5 5 4 6 5 5 07/30 Write block 98 len 24576 data: 4 5 4 4 5 7 08/30 Write block 539 len 8192 data: 8 10 09/30 Write block 306 len 16384 data: 9 9 8 8 10/30 Write block 410 len 12288 data: 6 6 7 11/30 Write block 120 len 32768 data: 5 6 7 7 7 6 6 6 12/30 Read block 235 len 20480 13/30 Read block 430 len 4096 14/30 Write block 495 len 36864 data: 3 4 6 6 6 4 3 4 3 15/30 Read block 207 len 40960 16/30 Write block 456 len 16384 data: 6 6 6 5 17/30 Write block 143 len 20480 data: 6 5 7 8 6 18/30 Write block 117 len 36864 data: 5 7 6 6 7 8 8 8 7 19/30 Read block 509 len 32768 20/30 Read block 233 len 16384 21/30 Read block 237 len 12288 22/30 Read block 242 len 24576 23/30 Write block 43 len 32768 data: 7 8 8 8 9 9 8 7 24/30 Read block 332 len 24576 25/30 Read block 287 len 32768 26/30 Read block 194 len 8192 27/30 Write block 287 len 8192 data: 5 5 28/30 Write block 545 len 32768 data: 7 7 7 9 10 11 10 10 29/30 Write block 583 len 28672 data: 10 6 7 6 5 5 6 30/30 Read block 316 len 8192 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 5 Acked 1004 Write 3 Done Done Done false 6 Acked 1005 Write 9 Done Done Done false 7 Acked 1006 Write 6 Done Done Done false 8 Acked 1007 Write 2 Done Done Done false 9 Acked 1008 Write 4 Done Done Done false 10 Acked 1009 Write 3 Done Done Done false 11 Acked 1010 Write 8 Done Done Done false 12 Acked 1011 Read 5 Done Done Done false 13 Acked 1012 Read 1 Done Done Done false 14 Acked 1013 Write 9 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Write 4 Done Done Done false 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:20.777405321Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 102 102 102 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 58 58 58 103 103 103 T T T 3 060-079 56 56 56 101 101 101 F F F 4 080-099 58 58 58 103 103 103 T T T 5 100-119 59 59 59 104 104 104 T T T 6 120-139 59 59 59 104 104 104 T T T 7 140-159 54 54 54 99 99 99 T T T 8 160-179 52 52 52 96 96 96 F F F 9 180-199 52 52 52 95 95 95 T T T 10 200-219 59 59 59 104 104 104 T T T 11 220-239 59 59 59 104 104 104 F F F 12 240-259 56 56 56 101 101 101 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 53 53 53 97 97 97 T T T 15 300-319 59 59 59 104 104 104 T T T 16 320-339 58 58 58 103 103 103 F F F 17 340-359 58 58 58 103 103 103 F F F 18 360-379 56 56 56 101 101 101 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 55 55 55 100 100 100 T T T 21 420-439 50 50 50 92 92 92 F F F 22 440-459 59 59 59 104 104 104 T T T 23 460-479 59 59 59 104 104 104 F F F 24 480-499 58 58 58 103 103 103 T T T 25 500-519 60 60 60 105 105 105 T T T 26 520-539 56 56 56 101 101 101 T T T 27 540-559 52 52 52 95 95 95 T T T 28 560-579 60 60 60 105 105 105 F F F 29 580-599 60 60 60 105 105 105 T T T Max gen: 60, Max flush: 105 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 102 102 102 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 61 61 61 106 106 106 F F F 3 060-079 56 56 56 101 101 101 F F F 4 080-099 61 61 61 106 106 106 F F F 5 100-119 61 61 61 106 106 106 F F F 6 120-139 61 61 61 106 106 106 F F F 7 140-159 61 61 61 106 106 106 F F F 8 160-179 52 52 52 96 96 96 F F F 9 180-199 61 61 61 106 106 106 F F F 10 200-219 61 61 61 106 106 106 F F F 11 220-239 59 59 59 104 104 104 F F F 12 240-259 56 56 56 101 101 101 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 61 61 61 106 106 106 F F F 15 300-319 61 61 61 106 106 106 F F F 16 320-339 58 58 58 103 103 103 F F F 17 340-359 58 58 58 103 103 103 F F F 18 360-379 56 56 56 101 101 101 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 61 61 61 106 106 106 F F F 23 460-479 59 59 59 104 104 104 F F F 24 480-499 61 61 61 106 106 106 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 61 61 61 106 106 106 F F F 27 540-559 61 61 61 106 106 106 F F F 28 560-579 60 60 60 105 105 105 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 61, Max flush: 106 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.252837136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:25.296501707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.297113931Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.329876179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.330775693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.330811576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.330831833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.33084693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430} {"msg":"Crucible ad337362-05af-4950-bc13-3fa1243e2318 has session id: baed88a7-b35d-4edb-9a84-fc2b7fd462d7","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.331355332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Upstairs opts: Upstairs UUID: ad337362-05af-4950-bc13-3fa1243e2318, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.331402004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Crucible stats registered with UUID: ad337362-05af-4950-bc13-3fa1243e2318","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.331419561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363043118Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430} {"msg":"ad337362-05af-4950-bc13-3fa1243e2318 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363112076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363138791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363174835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.36320099Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363225696Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363246572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363269358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363304062Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363336157Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363366211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51548)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363392087Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53744)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363416783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60657)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.363443618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.364401971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.364430416Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.364444954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.364463771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.364593858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.364659206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365085182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365111918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365377421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365759785Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365798638Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365825853Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365849219Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365876565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.3659026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.36596056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.365989575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[0]R flush_numbers[0..12]: [102, 103, 106, 101, 106, 106, 106, 106, 96, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366020849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[0]R generation[0..12]: [58, 58, 61, 56, 61, 61, 61, 61, 52, 61, 61, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366037067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366051244Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[1]R flush_numbers[0..12]: [102, 103, 106, 101, 106, 106, 106, 106, 96, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366065092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[1]R generation[0..12]: [58, 58, 61, 56, 61, 61, 61, 61, 52, 61, 61, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366078649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366092547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[2]R flush_numbers[0..12]: [102, 103, 106, 101, 106, 106, 106, 106, 96, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366106345Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[2]R generation[0..12]: [58, 58, 61, 56, 61, 61, 61, 61, 52, 61, 61, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366123292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366137449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366150487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366163915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366176752Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.36618962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"":"downstairs","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366203198Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366215236Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"0","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366239911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"1","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366253089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"client":"2","":"downstairs","session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366266217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366278425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"ad337362-05af-4950-bc13-3fa1243e2318 is now active with session: baed88a7-b35d-4edb-9a84-fc2b7fd462d7","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366291372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.36630332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430,"session_id":"baed88a7-b35d-4edb-9a84-fc2b7fd462d7"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:25.366315468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 551 len 16384 data: 11 11 7 7 02/30 Write block 159 len 8192 data: 5 6 03/30 Read block 183 len 20480 04/30 Write block 335 len 28672 data: 8 7 7 8 8 8 7 05/30 Read block 471 len 4096 06/30 Read block 560 len 8192 07/30 Read block 320 len 32768 08/30 Read block 468 len 28672 09/30 Write block 94 len 32768 data: 5 5 6 5 5 6 5 5 10/30 Write block 566 len 32768 data: 6 7 6 6 5 5 7 6 11/30 Read block 327 len 36864 12/30 Read block 582 len 40960 13/30 Read block 380 len 4096 14/30 Write block 121 len 28672 data: 8 9 9 9 8 7 7 15/30 Write block 163 len 40960 data: 6 5 4 6 6 8 6 5 6 6 16/30 Read block 420 len 40960 17/30 Write block 163 len 16384 data: 7 6 5 7 18/30 Write block 335 len 16384 data: 9 8 8 9 19/30 Flush 20/30 Write block 444 len 32768 data: 3 4 7 7 5 5 6 5 21/30 Read block 374 len 16384 22/30 Write block 10 len 40960 data: 7 8 10 11 8 9 8 9 9 10 23/30 Read block 252 len 8192 24/30 Read block 185 len 4096 25/30 Flush 26/30 Write block 133 len 16384 data: 4 6 5 6 27/30 Read block 81 len 16384 28/30 Read block 438 len 28672 29/30 Read block 341 len 20480 30/30 Read block 77 len 28672 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:25.393363541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 108 108 108 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 61 61 61 106 106 106 F F F 3 060-079 56 56 56 101 101 101 F F F 4 080-099 62 62 62 107 107 107 F F F 5 100-119 62 62 62 107 107 107 F F F 6 120-139 62 62 62 107 107 107 T T T 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 61 61 61 106 106 106 F F F 10 200-219 61 61 61 106 106 106 F F F 11 220-239 59 59 59 104 104 104 F F F 12 240-259 56 56 56 101 101 101 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 61 61 61 106 106 106 F F F 15 300-319 61 61 61 106 106 106 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 F F F 18 360-379 56 56 56 101 101 101 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 62 62 62 108 108 108 F F F 23 460-479 59 59 59 104 104 104 F F F 24 480-499 61 61 61 106 106 106 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 61 61 61 106 106 106 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 62, Max flush: 108 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 108 108 108 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 61 61 61 106 106 106 F F F 3 060-079 56 56 56 101 101 101 F F F 4 080-099 62 62 62 107 107 107 F F F 5 100-119 62 62 62 107 107 107 F F F 6 120-139 63 63 63 109 109 109 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 61 61 61 106 106 106 F F F 10 200-219 61 61 61 106 106 106 F F F 11 220-239 59 59 59 104 104 104 F F F 12 240-259 56 56 56 101 101 101 F F F 13 260-279 53 53 53 97 97 97 F F F 14 280-299 61 61 61 106 106 106 F F F 15 300-319 61 61 61 106 106 106 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 F F F 18 360-379 56 56 56 101 101 101 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 62 62 62 108 108 108 F F F 23 460-479 59 59 59 104 104 104 F F F 24 480-499 61 61 61 106 106 106 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 61 61 61 106 106 106 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 63, Max flush: 109 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.69460319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.738654754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.739622255Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.739667538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.739684995Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.739702661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438} {"msg":"Crucible 028bd9d7-4d40-4fa8-bc99-bcde76e713c4 has session id: c965f652-9b9c-480c-b18a-1f51200fb0ab","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.740168411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Upstairs opts: Upstairs UUID: 028bd9d7-4d40-4fa8-bc99-bcde76e713c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.740226461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Crucible stats registered with UUID: 028bd9d7-4d40-4fa8-bc99-bcde76e713c4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.740257155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771451877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438} {"msg":"028bd9d7-4d40-4fa8-bc99-bcde76e713c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771508697Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771539351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771590992Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771630496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771665969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771699113Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771729808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771754574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.771928014Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772071999Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:40725)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772103823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:51468)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.77212453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:53927)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772145356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772579271Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772614994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772801112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772855063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772894066Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.772920141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.773299345Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.773440061Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.773763434Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.773952332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.773979957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.773995164Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.77407655Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774097226Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774115363Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.77442024Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774441396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[0]R flush_numbers[0..12]: [108, 103, 106, 101, 107, 107, 109, 107, 107, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774466032Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[0]R generation[0..12]: [62, 58, 61, 56, 62, 62, 63, 62, 62, 61, 61, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774480349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774494747Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[1]R flush_numbers[0..12]: [108, 103, 106, 101, 107, 107, 109, 107, 107, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774508565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[1]R generation[0..12]: [62, 58, 61, 56, 62, 62, 63, 62, 62, 61, 61, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774522062Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.77453626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[2]R flush_numbers[0..12]: [108, 103, 106, 101, 107, 107, 109, 107, 107, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774550148Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[2]R generation[0..12]: [62, 58, 61, 56, 62, 62, 63, 62, 62, 61, 61, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774563595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774577283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774590241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774603348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774620625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774634193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"":"downstairs","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774647411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:12:29.774659349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"0","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774682904Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"1","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774695872Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"client":"2","":"downstairs","session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.7747088Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774721058Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"028bd9d7-4d40-4fa8-bc99-bcde76e713c4 is now active with session: c965f652-9b9c-480c-b18a-1f51200fb0ab","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774733896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774745793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438,"session_id":"c965f652-9b9c-480c-b18a-1f51200fb0ab"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:29.774757941Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 84 len 16384 data: 8 6 6 6 02/30 Read block 484 len 40960 03/30 Write block 451 len 32768 data: 6 5 6 6 5 7 7 7 04/30 Write block 449 len 24576 data: 6 7 7 6 7 7 05/30 Write block 76 len 12288 data: 7 5 5 06/30 Write block 407 len 20480 data: 5 6 5 7 7 07/30 Write block 248 len 8192 data: 4 4 08/30 Flush 09/30 Write block 368 len 32768 data: 6 7 8 9 10 10 10 9 10/30 Write block 281 len 16384 data: 6 7 6 4 11/30 Read block 343 len 24576 12/30 Flush 13/30 Write block 523 len 32768 data: 9 9 8 4 3 4 8 9 14/30 Read block 301 len 32768 15/30 Write block 184 len 8192 data: 5 6 16/30 Write block 276 len 20480 data: 7 6 6 6 6 17/30 Write block 50 len 40960 data: 8 7 7 6 7 5 4 6 6 4 18/30 Flush 19/30 Write block 16 len 4096 data: 9 20/30 Read block 542 len 28672 21/30 Write block 209 len 28672 data: 6 6 6 4 6 6 3 22/30 Write block 480 len 36864 data: 4 3 2 4 4 4 4 4 4 23/30 Read block 266 len 32768 24/30 Read block 66 len 12288 25/30 Write block 461 len 28672 data: 4 5 6 7 6 5 4 26/30 Read block 302 len 4096 27/30 Read block 365 len 28672 28/30 Read block 167 len 28672 29/30 Read block 315 len 24576 30/30 Read block 116 len 16384 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:29.800625939Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 108 108 108 T T T 1 020-039 58 58 58 103 103 103 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 64 64 64 110 110 110 F F F 4 080-099 64 64 64 110 110 110 F F F 5 100-119 62 62 62 107 107 107 F F F 6 120-139 63 63 63 109 109 109 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 64 64 64 112 112 112 F F F 10 200-219 61 61 61 106 106 106 T T T 11 220-239 59 59 59 104 104 104 F F F 12 240-259 64 64 64 110 110 110 F F F 13 260-279 64 64 64 112 112 112 F F F 14 280-299 64 64 64 112 112 112 F F F 15 300-319 61 61 61 106 106 106 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 F F F 18 360-379 64 64 64 111 111 111 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 59 59 59 104 104 104 T T T 24 480-499 61 61 61 106 106 106 T T T 25 500-519 61 61 61 106 106 106 F F F 26 520-539 64 64 64 112 112 112 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 64, Max flush: 112 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 64 64 64 110 110 110 F F F 4 080-099 64 64 64 110 110 110 F F F 5 100-119 62 62 62 107 107 107 F F F 6 120-139 63 63 63 109 109 109 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 64 64 64 112 112 112 F F F 10 200-219 65 65 65 113 113 113 F F F 11 220-239 59 59 59 104 104 104 F F F 12 240-259 64 64 64 110 110 110 F F F 13 260-279 64 64 64 112 112 112 F F F 14 280-299 64 64 64 112 112 112 F F F 15 300-319 61 61 61 106 106 106 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 F F F 18 360-379 64 64 64 111 111 111 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 64 64 64 110 110 110 F F F 21 420-439 50 50 50 92 92 92 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 65 65 65 113 113 113 F F F 24 480-499 65 65 65 113 113 113 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 64 64 64 112 112 112 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 62 62 62 107 107 107 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 65, Max flush: 113 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.103136737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:34.146854649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.147408892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.179578894Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.180363958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.180401261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.180415539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.180429676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446} {"msg":"Crucible 1587d781-192e-4b6b-b0e2-14454c71b171 has session id: 49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.180897945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Upstairs opts: Upstairs UUID: 1587d781-192e-4b6b-b0e2-14454c71b171, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.180945447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Crucible stats registered with UUID: 1587d781-192e-4b6b-b0e2-14454c71b171","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.180972532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212157965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446} {"msg":"1587d781-192e-4b6b-b0e2-14454c71b171 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212226433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212260827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212307549Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212349671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212385655Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212422749Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212451494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212470061Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212488187Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212511253Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_connection connected from Ok(127.0.0.1:50724)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.21264622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_connection connected from Ok(127.0.0.1:40058)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.212888698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_connection connected from Ok(127.0.0.1:63537)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.213021185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.213434973Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.213464927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.213479965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.213493542Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.213771894Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.213812477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.214323918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.214354833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.214371899Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.214936991Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.214969096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.214984353Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.214997171Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215011048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215024176Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215039503Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215053041Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[0]R flush_numbers[0..12]: [113, 103, 112, 110, 110, 107, 109, 107, 107, 112, 113, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215066739Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[0]R generation[0..12]: [65, 58, 64, 64, 64, 62, 63, 62, 62, 64, 65, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215080806Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215095124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[1]R flush_numbers[0..12]: [113, 103, 112, 110, 110, 107, 109, 107, 107, 112, 113, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215108841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[1]R generation[0..12]: [65, 58, 64, 64, 64, 62, 63, 62, 62, 64, 65, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215122299Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215136166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[2]R flush_numbers[0..12]: [113, 103, 112, 110, 110, 107, 109, 107, 107, 112, 113, 104]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215149654Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[2]R generation[0..12]: [65, 58, 64, 64, 64, 62, 63, 62, 62, 64, 65, 59]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215162902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.21517661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215189387Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215202395Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215215273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.21523007Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"":"downstairs","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"No downstairs reconciliation required",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215253976Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215266054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"0","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215279352Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"1","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215292899Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"client":"2","":"downstairs","session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215306717Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215319115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"1587d781-192e-4b6b-b0e2-14454c71b171 is now active with session: 49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215331712Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215343601Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446,"session_id":"49c96ec6-0ce9-4bec-8fa8-3fa7f2c6cf84"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:34.215355708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 468 len 8192 data: 5 3 02/30 Flush 03/30 Write block 232 len 12288 data: 4 5 6 04/30 Read block 257 len 36864 05/30 Read block 540 len 40960 06/30 Read block 239 len 8192 07/30 Write block 415 len 12288 data: 8 6 6 08/30 Flush 09/30 Write block 478 len 40960 data: 4 4 5 4 3 5 5 5 5 5 10/30 Write block 124 len 36864 data: 10 9 8 8 5 5 5 5 5 11/30 Read block 225 len 32768 12/30 Read block 427 len 36864 13/30 Write block 412 len 32768 data: 8 8 7 9 7 7 4 4 14/30 Write block 243 len 40960 data: 4 2 2 2 2 5 5 5 5 4 15/30 Flush 16/30 Write block 197 len 32768 data: 6 6 6 6 5 7 6 6 17/30 Write block 116 len 16384 data: 5 6 8 7 18/30 Write block 180 len 32768 data: 5 4 5 5 6 7 7 7 19/30 Write block 469 len 4096 data: 4 20/30 Read block 378 len 8192 21/30 Write block 116 len 8192 data: 6 7 22/30 Write block 419 len 28672 data: 5 4 4 4 4 4 5 23/30 Read block 540 len 36864 24/30 Write block 401 len 28672 data: 4 5 4 5 5 6 6 25/30 Write block 358 len 40960 data: 3 5 5 6 9 9 10 8 8 8 26/30 Write block 571 len 20480 data: 6 8 7 7 8 27/30 Write block 74 len 12288 data: 6 6 8 28/30 Write block 93 len 40960 data: 3 6 6 7 6 6 7 6 6 6 29/30 Write block 192 len 8192 data: 5 3 30/30 Write block 196 len 36864 data: 7 7 7 7 7 6 8 7 7 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 8 Done Done Done false 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Write 3 Done Sent Done false 28 Acked 1027 Write 10 Sent Sent Sent false 29 Acked 1028 Write 2 Sent Sent Sent false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 4 3 10 Done 12 11 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:34.239497927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 64 64 64 110 110 110 T T T 4 080-099 64 64 64 110 110 110 T T T 5 100-119 62 62 62 107 107 107 T T T 6 120-139 66 66 66 116 116 116 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 64 64 64 112 112 112 T T T 10 200-219 65 65 65 113 113 113 T T T 11 220-239 66 66 66 115 115 115 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 64 64 64 112 112 112 F F F 14 280-299 64 64 64 112 112 112 F F F 15 300-319 61 61 61 106 106 106 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 T T T 18 360-379 64 64 64 111 111 111 T T T 19 380-399 55 55 55 100 100 100 F F F 20 400-419 66 66 66 116 116 116 T T T 21 420-439 50 50 50 92 92 92 T T T 22 440-459 64 64 64 110 110 110 F F F 23 460-479 66 66 66 116 116 116 T T T 24 480-499 66 66 66 116 116 116 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 64 64 64 112 112 112 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 62 62 62 107 107 107 T T T 29 580-599 61 61 61 106 106 106 F F F Max gen: 66, Max flush: 116 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 67 67 67 117 117 117 F F F 4 080-099 67 67 67 117 117 117 F F F 5 100-119 67 67 67 117 117 117 F F F 6 120-139 66 66 66 116 116 116 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 67 67 67 117 117 117 F F F 11 220-239 66 66 66 115 115 115 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 64 64 64 112 112 112 F F F 14 280-299 64 64 64 112 112 112 F F F 15 300-319 61 61 61 106 106 106 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 67 67 67 117 117 117 F F F 18 360-379 67 67 67 117 117 117 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 67 67 67 117 117 117 F F F 21 420-439 67 67 67 117 117 117 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 67 67 67 117 117 117 F F F 24 480-499 66 66 66 116 116 116 F F F 25 500-519 61 61 61 106 106 106 F F F 26 520-539 64 64 64 112 112 112 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 67, Max flush: 117 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.674554519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:38.717481148Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.71810538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.750154063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.751100398Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.751139571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.751153809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.751168076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454} {"msg":"Crucible 07ec99b7-79b5-48a4-96d5-3aa115fafa68 has session id: 69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.751638924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Upstairs opts: Upstairs UUID: 07ec99b7-79b5-48a4-96d5-3aa115fafa68, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.751687786Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Crucible stats registered with UUID: 07ec99b7-79b5-48a4-96d5-3aa115fafa68","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.751712812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.782786984Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454} {"msg":"07ec99b7-79b5-48a4-96d5-3aa115fafa68 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.782853002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.782886606Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.782931109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.782964453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783026972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783063915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783098409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783132264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783218469Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783246584Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:55932)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783294606Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:39759)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783365163Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63288)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.783407066Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784020639Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784061252Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784088137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784111293Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784136529Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784161115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784718658Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784825919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.784860843Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785429234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785456079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785470397Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785483165Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785497982Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785515799Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785528757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785541994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[0]R flush_numbers[0..12]: [113, 103, 112, 117, 117, 117, 116, 107, 107, 117, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785555782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[0]R generation[0..12]: [65, 58, 64, 67, 67, 67, 66, 62, 62, 67, 67, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785569659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785583437Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[1]R flush_numbers[0..12]: [113, 103, 112, 117, 117, 117, 116, 107, 107, 117, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785597135Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[1]R generation[0..12]: [65, 58, 64, 67, 67, 67, 66, 62, 62, 67, 67, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785610522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.78562433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[2]R flush_numbers[0..12]: [113, 103, 112, 117, 117, 117, 116, 107, 107, 117, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785637998Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[2]R generation[0..12]: [65, 58, 64, 67, 67, 67, 66, 62, 62, 67, 67, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785651275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785664983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785677751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785690658Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:12:38.785705556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785728172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"":"downstairs","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.78574122Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785753087Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"0","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785766155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"1","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785779403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"client":"2","":"downstairs","session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785792491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785804649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"07ec99b7-79b5-48a4-96d5-3aa115fafa68 is now active with session: 69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785817466Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:38.785829444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454,"session_id":"69c0fa22-fa64-4bc8-bdb5-e99f44cd67a4"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:12:38.785841502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 8192 data: 7 8 02/30 Write block 138 len 4096 data: 7 03/30 Write block 203 len 12288 data: 8 8 3 04/30 Write block 245 len 4096 data: 3 05/30 Write block 109 len 24576 data: 6 6 5 5 3 5 06/30 Read block 115 len 36864 07/30 Read block 526 len 12288 08/30 Flush 09/30 Write block 118 len 16384 data: 9 8 7 9 10/30 Read block 374 len 20480 11/30 Read block 176 len 36864 12/30 Read block 102 len 40960 13/30 Write block 338 len 36864 data: 10 9 9 8 6 5 6 2 3 14/30 Write block 481 len 24576 data: 5 4 6 6 6 6 15/30 Read block 337 len 12288 16/30 Read block 534 len 8192 17/30 Write block 401 len 36864 data: 5 6 5 6 6 7 7 7 6 18/30 Write block 538 len 16384 data: 7 9 11 9 19/30 Flush 20/30 Write block 466 len 16384 data: 6 5 6 5 21/30 Write block 333 len 24576 data: 5 6 10 9 9 11 22/30 Write block 219 len 20480 data: 5 4 3 4 3 23/30 Write block 431 len 32768 data: 3 3 3 5 5 2 2 2 24/30 Write block 331 len 8192 data: 5 5 25/30 Write block 508 len 4096 data: 7 26/30 Write block 303 len 16384 data: 3 5 7 10 27/30 Write block 425 len 20480 data: 6 4 4 5 4 28/30 Read block 455 len 20480 29/30 Read block 536 len 8192 30/30 Write block 96 len 8192 data: 8 7 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:38.808351335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 67 67 67 117 117 117 F F F 4 080-099 68 68 68 118 118 118 T T T 5 100-119 68 68 68 119 119 119 F F F 6 120-139 68 68 68 119 119 119 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 68 68 68 118 118 118 T T T 11 220-239 66 66 66 115 115 115 T T T 12 240-259 68 68 68 118 118 118 F F F 13 260-279 64 64 64 112 112 112 F F F 14 280-299 64 64 64 112 112 112 F F F 15 300-319 61 61 61 106 106 106 T T T 16 320-339 68 68 68 119 119 119 T T T 17 340-359 68 68 68 119 119 119 F F F 18 360-379 67 67 67 117 117 117 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 67 67 67 117 117 117 T T T 22 440-459 64 64 64 110 110 110 F F F 23 460-479 67 67 67 117 117 117 T T T 24 480-499 68 68 68 119 119 119 F F F 25 500-519 61 61 61 106 106 106 T T T 26 520-539 68 68 68 119 119 119 F F F 27 540-559 68 68 68 119 119 119 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 68, Max flush: 119 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 58 58 58 103 103 103 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 67 67 67 117 117 117 F F F 4 080-099 69 69 69 120 120 120 F F F 5 100-119 68 68 68 119 119 119 F F F 6 120-139 68 68 68 119 119 119 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 69 69 69 120 120 120 F F F 11 220-239 69 69 69 120 120 120 F F F 12 240-259 68 68 68 118 118 118 F F F 13 260-279 64 64 64 112 112 112 F F F 14 280-299 64 64 64 112 112 112 F F F 15 300-319 69 69 69 120 120 120 F F F 16 320-339 69 69 69 120 120 120 F F F 17 340-359 68 68 68 119 119 119 F F F 18 360-379 67 67 67 117 117 117 F F F 19 380-399 55 55 55 100 100 100 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 69 69 69 120 120 120 F F F 22 440-459 64 64 64 110 110 110 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 68 68 68 119 119 119 F F F 25 500-519 69 69 69 120 120 120 F F F 26 520-539 68 68 68 119 119 119 F F F 27 540-559 68 68 68 119 119 119 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 69, Max flush: 120 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.204745675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.24936275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.250266143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.250300497Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.250314465Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.250328612Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462} {"msg":"Crucible bebd8726-dc3f-47bb-b03a-7de3d2888e9a has session id: 3d67313b-7ca5-4f6f-8893-362ebc6b6cee","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.250843053Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Upstairs opts: Upstairs UUID: bebd8726-dc3f-47bb-b03a-7de3d2888e9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.250886355Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Crucible stats registered with UUID: bebd8726-dc3f-47bb-b03a-7de3d2888e9a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.250911191Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282112271Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462} {"msg":"bebd8726-dc3f-47bb-b03a-7de3d2888e9a active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282154614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282172981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282203056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282221582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.2822384Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282341331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282375166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282394092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282422357Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282451582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_connection connected from Ok(127.0.0.1:38324)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282479238Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_connection connected from Ok(127.0.0.1:43615)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282505663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_connection connected from Ok(127.0.0.1:47116)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.282653867Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.283417254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.283454178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.283490751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.283543522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.283567758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.283582296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284022569Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284120142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.28430099Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284630523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284655929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284670446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284772899Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284802473Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284818331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.28493747Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284957456Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[0]R flush_numbers[0..12]: [113, 103, 112, 117, 120, 119, 119, 107, 107, 117, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.284978623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[0]R generation[0..12]: [65, 58, 64, 67, 69, 68, 68, 62, 62, 67, 69, 69]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.28499344Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285007638Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[1]R flush_numbers[0..12]: [113, 103, 112, 117, 120, 119, 119, 107, 107, 117, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285021505Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[1]R generation[0..12]: [65, 58, 64, 67, 69, 68, 68, 62, 62, 67, 69, 69]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285034813Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285048531Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[2]R flush_numbers[0..12]: [113, 103, 112, 117, 120, 119, 119, 107, 107, 117, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285062438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[2]R generation[0..12]: [65, 58, 64, 67, 69, 68, 68, 62, 62, 67, 69, 69]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285075826Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285089483Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285102371Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285116159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285129206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285142594Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"":"downstairs","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285155712Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285169649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","Wait for a query_work_queue command to finish before sending IO pid":1462,"client":"0","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285194285Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"1","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285207253Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"client":"2","":"downstairs","session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.28522037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285232648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"bebd8726-dc3f-47bb-b03a-7de3d2888e9a is now active with session: 3d67313b-7ca5-4f6f-8893-362ebc6b6cee","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285245256Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285257194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462,"session_id":"3d67313b-7ca5-4f6f-8893-362ebc6b6cee"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:43.285269472Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 393 len 8192 data: 4 4 02/30 Read block 407 len 16384 03/30 Read block 364 len 4096 04/30 Flush 05/30 Flush 06/30 Write block 513 len 12288 data: 8 7 10 07/30 Read block 367 len 40960 08/30 Write block 233 len 24576 data: 6 7 8 7 7 7 09/30 Read block 53 len 36864 10/30 Read block 208 len 8192 11/30 Write block 89 len 20480 data: 4 4 3 3 4 12/30 Write block 241 len 28672 data: 6 5 5 3 4 3 3 13/30 Write block 118 len 8192 data: 10 9 14/30 Write block 430 len 12288 data: 3 4 4 15/30 Write block 449 len 8192 data: 7 8 16/30 Read block 589 len 12288 17/30 Write block 501 len 16384 data: 4 5 4 3 18/30 Read block 588 len 40960 19/30 Read block 397 len 8192 20/30 Write block 209 len 4096 data: 7 21/30 Flush 22/30 Write block 29 len 24576 data: 4 5 6 6 7 7 23/30 Write block 53 len 8192 data: 7 8 24/30 Read block 181 len 16384 25/30 Write block 375 len 36864 data: 10 7 5 5 6 5 6 6 8 26/30 Write block 285 len 32768 data: 5 6 6 6 6 3 3 3 27/30 Write block 491 len 28672 data: 3 3 2 3 4 5 7 28/30 Write block 71 len 24576 data: 9 10 6 7 7 9 29/30 Read block 161 len 12288 30/30 Write block 277 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 3 Done Done Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:43.308657042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 58 58 58 103 103 103 T T T 2 040-059 64 64 64 112 112 112 T T T 3 060-079 67 67 67 117 117 117 T T T 4 080-099 70 70 70 123 123 123 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 68 68 68 119 119 119 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 70 70 70 123 123 123 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 70 70 70 123 123 123 F F F 13 260-279 64 64 64 112 112 112 T T T 14 280-299 64 64 64 112 112 112 T T T 15 300-319 69 69 69 120 120 120 F F F 16 320-339 69 69 69 120 120 120 F F F 17 340-359 68 68 68 119 119 119 F F F 18 360-379 67 67 67 117 117 117 T T T 19 380-399 70 70 70 121 121 121 T T T 20 400-419 68 68 68 119 119 119 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 68 68 68 119 119 119 T T T 25 500-519 70 70 70 123 123 123 F F F 26 520-539 68 68 68 119 119 119 F F F 27 540-559 68 68 68 119 119 119 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 70, Max flush: 123 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 71 71 71 124 124 124 F F F 3 060-079 71 71 71 124 124 124 F F F 4 080-099 70 70 70 123 123 123 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 68 68 68 119 119 119 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 67 67 67 117 117 117 F F F 10 200-219 70 70 70 123 123 123 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 70 70 70 123 123 123 F F F 13 260-279 71 71 71 124 124 124 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 120 120 120 F F F 16 320-339 69 69 69 120 120 120 F F F 17 340-359 68 68 68 119 119 119 F F F 18 360-379 71 71 71 124 124 124 F F F 19 380-399 71 71 71 124 124 124 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 71 71 71 124 124 124 F F F 25 500-519 70 70 70 123 123 123 F F F 26 520-539 68 68 68 119 119 119 F F F 27 540-559 68 68 68 119 119 119 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 71, Max flush: 124 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.67098633Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.715441403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.716311702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.716348936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.716363363Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.716377711Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470} {"msg":"Crucible 7206096a-aab7-493f-8813-a85dd30700ef has session id: dc021d85-8b19-4b23-a0b8-77bafcef4fb1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.716800427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Upstairs opts: Upstairs UUID: 7206096a-aab7-493f-8813-a85dd30700ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.716848199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Crucible stats registered with UUID: 7206096a-aab7-493f-8813-a85dd30700ef","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.716873344Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747703879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470} {"msg":"7206096a-aab7-493f-8813-a85dd30700ef active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747783325Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747803302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747834026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747859762Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747879119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747896246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747913163Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.74792962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.747954865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.74798544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51535)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.748016275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42778)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.748167088Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40640)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.748244645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.748939594Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.748987446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.749065112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.749097467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.749244151Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.749286834Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.749682595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.74971184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750017737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750287479Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750308756Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750322544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.75039866Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750417297Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750430785Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750590507Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750610413Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[0]R flush_numbers[0..12]: [113, 124, 124, 124, 123, 123, 119, 107, 107, 117, 123, 123]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750637758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[0]R generation[0..12]: [65, 71, 71, 71, 70, 70, 68, 62, 62, 67, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750652006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750666204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[1]R flush_numbers[0..12]: [113, 124, 124, 124, 123, 123, 119, 107, 107, 117, 123, 123]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750683351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[1]R generation[0..12]: [65, 71, 71, 71, 70, 70, 68, 62, 62, 67, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750697038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750710946Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[2]R flush_numbers[0..12]: [113, 124, 124, 124, 123, 123, 119, 107, 107, 117, 123, 123]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750724543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[2]R generation[0..12]: [65, 71, 71, 71, 70, 70, 68, 62, 62, 67, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750737911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750751519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750764267Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750777344Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750790242Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.75080301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"":"downstairs","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750815947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750827905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"0","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750841283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"1","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750864489Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"client":"2","":"downstairs","session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750877897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750892914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"7206096a-aab7-493f-8813-a85dd30700ef is now active with session: dc021d85-8b19-4b23-a0b8-77bafcef4fb1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750905882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.75091809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470,"session_id":"dc021d85-8b19-4b23-a0b8-77bafcef4fb1"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:47.750930748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 339 len 24576 data: 10 10 9 7 6 7 02/30 Write block 348 len 32768 data: 3 3 4 3 3 4 3 3 03/30 Write block 504 len 4096 data: 4 04/30 Read block 94 len 40960 05/30 Write block 389 len 20480 data: 5 4 6 4 5 06/30 Write block 151 len 4096 data: 4 07/30 Read block 93 len 20480 08/30 Write block 422 len 12288 data: 5 5 5 09/30 Read block 321 len 20480 10/30 Read block 339 len 40960 11/30 Write block 367 len 24576 data: 9 7 8 9 10 11 12/30 Write block 137 len 24576 data: 7 8 8 6 5 7 13/30 Read block 8 len 32768 14/30 Read block 576 len 20480 15/30 Write block 198 len 16384 data: 8 8 8 7 16/30 Read block 505 len 24576 17/30 Write block 53 len 4096 data: 8 18/30 Write block 127 len 24576 data: 9 6 6 6 6 6 19/30 Write block 73 len 12288 data: 7 8 8 20/30 Read block 440 len 8192 21/30 Read block 141 len 28672 22/30 Read block 202 len 8192 23/30 Read block 285 len 36864 24/30 Read block 407 len 40960 25/30 Read block 431 len 24576 26/30 Read block 151 len 28672 27/30 Read block 333 len 12288 28/30 Read block 177 len 32768 29/30 Write block 195 len 28672 data: 6 8 8 9 9 9 8 30/30 Write block 144 len 36864 data: 6 8 9 7 6 5 5 5 5 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Write 8 Done Done Done false 3 Acked 1002 Write 1 Done Done Done false 4 Acked 1003 Read 10 Done Done Done false 5 Acked 1004 Write 5 Done Done Done false 6 Acked 1005 Write 1 Done Done Done false 7 Acked 1006 Read 5 Done Done Done false 8 Acked 1007 Write 3 Done Done Done false 9 Acked 1008 Read 5 Done Done Done false 10 Acked 1009 Read 10 Done Done Done false 11 Acked 1010 Write 6 Done Done Done false 12 Acked 1011 Write 6 Done Done Done false 13 Acked 1012 Read 8 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Write 4 Done Done Done false 16 Acked 1015 Read 6 Done Done Done false 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 7 Sent Sent Sent false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:47.770597665Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 71 71 71 124 124 124 T T T 3 060-079 71 71 71 124 124 124 T T T 4 080-099 70 70 70 123 123 123 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 68 68 68 119 119 119 T T T 7 140-159 62 62 62 107 107 107 T T T 8 160-179 62 62 62 107 107 107 F F F 9 180-199 67 67 67 117 117 117 T T T 10 200-219 70 70 70 123 123 123 T T T 11 220-239 70 70 70 123 123 123 F F F 12 240-259 70 70 70 123 123 123 F F F 13 260-279 71 71 71 124 124 124 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 120 120 120 F F F 16 320-339 69 69 69 120 120 120 T T T 17 340-359 68 68 68 119 119 119 T T T 18 360-379 71 71 71 124 124 124 T T T 19 380-399 71 71 71 124 124 124 T T T 20 400-419 68 68 68 119 119 119 F F F 21 420-439 70 70 70 123 123 123 T T T 22 440-459 70 70 70 123 123 123 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 71 71 71 124 124 124 F F F 25 500-519 70 70 70 123 123 123 T T T 26 520-539 68 68 68 119 119 119 F F F 27 540-559 68 68 68 119 119 119 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 71, Max flush: 124 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 72 72 72 125 125 125 F F F 3 060-079 72 72 72 125 125 125 F F F 4 080-099 70 70 70 123 123 123 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 72 72 72 125 125 125 F F F 7 140-159 72 72 72 125 125 125 F F F 8 160-179 62 62 62 107 107 107 F F F 9 180-199 72 72 72 125 125 125 F F F 10 200-219 72 72 72 125 125 125 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 70 70 70 123 123 123 F F F 13 260-279 71 71 71 124 124 124 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 120 120 120 F F F 16 320-339 72 72 72 125 125 125 F F F 17 340-359 72 72 72 125 125 125 F F F 18 360-379 72 72 72 125 125 125 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 72 72 72 125 125 125 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 71 71 71 124 124 124 F F F 25 500-519 72 72 72 125 125 125 F F F 26 520-539 68 68 68 119 119 119 F F F 27 540-559 68 68 68 119 119 119 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 72, Max flush: 125 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.189389527Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.233607492Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.234547919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.234593301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.234610868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.234628734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478} {"msg":"Crucible f7163606-5a7c-4485-8c22-3f678310f462 has session id: 2032a07f-5491-4abe-92e4-438e10c078f0","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.235189887Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Upstairs opts: Upstairs UUID: f7163606-5a7c-4485-8c22-3f678310f462, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.235236029Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Crucible stats registered with UUID: f7163606-5a7c-4485-8c22-3f678310f462","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.235254376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267451953Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478} {"msg":"f7163606-5a7c-4485-8c22-3f678310f462 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267503284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267528119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267560664Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.26757932Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267596448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267612775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267630411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267646849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267664396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267681493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54273)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267815059Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:47351)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.26786698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61088)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.267902864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.268784621Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.268812436Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.268826873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.268841041Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.268914018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.268944853Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.269544449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.269575543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.26959291Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270203744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270233578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270248416Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270260764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270274022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270287309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270299517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270313035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[0]R flush_numbers[0..12]: [113, 124, 125, 125, 123, 123, 125, 125, 107, 125, 125, 123]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270326782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[0]R generation[0..12]: [65, 71, 72, 72, 70, 70, 72, 72, 62, 72, 72, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.27034069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270359027Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[1]R flush_numbers[0..12]: [113, 124, 125, 125, 123, 123, 125, 125, 107, 125, 125, 123]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270374744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[1]R generation[0..12]: [65, 71, 72, 72, 70, 70, 72, 72, 62, 72, 72, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270388362Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270402169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[2]R flush_numbers[0..12]: [113, 124, 125, 125, 123, 123, 125, 125, 107, 125, 125, 123]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270415987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[2]R generation[0..12]: [65, 71, 72, 72, 70, 70, 72, 72, 62, 72, 72, 70]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270429324Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270443252Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Max found gen is 73","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.27045595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Generation requested: 74 >= found:73","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270469237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:12:52.270482125Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.27051221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"":"downstairs","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270538576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270554473Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"0","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270578429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"1","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270601485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"client":"2","":"downstairs","session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:12:52.27062735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270646537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"f7163606-5a7c-4485-8c22-3f678310f462 is now active with session: 2032a07f-5491-4abe-92e4-438e10c078f0","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270660164Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.270672022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478,"session_id":"2032a07f-5491-4abe-92e4-438e10c078f0"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:52.27068421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 361 len 8192 data: 7 10 02/30 Read block 438 len 4096 03/30 Write block 448 len 12288 data: 6 8 9 04/30 Write block 332 len 28672 data: 6 6 7 11 10 10 12 05/30 Read block 461 len 24576 06/30 Read block 201 len 36864 07/30 Read block 149 len 32768 08/30 Read block 279 len 32768 09/30 Write block 213 len 28672 data: 7 7 4 5 5 5 6 10/30 Read block 324 len 36864 11/30 Flush 12/30 Write block 335 len 4096 data: 12 13/30 Read block 355 len 12288 14/30 Read block 205 len 12288 15/30 Flush 16/30 Write block 290 len 20480 data: 4 4 4 3 3 17/30 Read block 151 len 28672 18/30 Write block 174 len 8192 data: 5 4 19/30 Read block 211 len 36864 20/30 Write block 110 len 36864 data: 7 6 6 4 6 4 7 8 11 21/30 Write block 216 len 40960 data: 6 6 6 7 5 4 5 4 5 6 22/30 Write block 546 len 32768 data: 8 8 10 11 12 12 12 8 23/30 Read block 251 len 4096 24/30 Write block 514 len 36864 data: 8 11 13 12 9 8 8 9 10 25/30 Read block 295 len 24576 26/30 Read block 14 len 12288 27/30 Read block 23 len 20480 28/30 Read block 162 len 12288 29/30 Write block 529 len 4096 data: 9 30/30 Write block 355 len 36864 data: 4 3 2 4 6 6 8 11 10 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 5 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Read 9 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Write 10 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:52.294543748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 72 72 72 125 125 125 F F F 3 060-079 72 72 72 125 125 125 F F F 4 080-099 70 70 70 123 123 123 F F F 5 100-119 70 70 70 123 123 123 T T T 6 120-139 72 72 72 125 125 125 F F F 7 140-159 72 72 72 125 125 125 F F F 8 160-179 62 62 62 107 107 107 T T T 9 180-199 72 72 72 125 125 125 F F F 10 200-219 74 74 74 126 126 126 T T T 11 220-239 70 70 70 123 123 123 T T T 12 240-259 70 70 70 123 123 123 F F F 13 260-279 71 71 71 124 124 124 F F F 14 280-299 71 71 71 124 124 124 T T T 15 300-319 69 69 69 120 120 120 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 72 72 72 125 125 125 T T T 18 360-379 74 74 74 126 126 126 T T T 19 380-399 72 72 72 125 125 125 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 72 72 72 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 71 71 71 124 124 124 F F F 25 500-519 72 72 72 125 125 125 T T T 26 520-539 68 68 68 119 119 119 T T T 27 540-559 68 68 68 119 119 119 T T T 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 74, Max flush: 127 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 72 72 72 125 125 125 F F F 3 060-079 72 72 72 125 125 125 F F F 4 080-099 70 70 70 123 123 123 F F F 5 100-119 75 75 75 128 128 128 F F F 6 120-139 72 72 72 125 125 125 F F F 7 140-159 72 72 72 125 125 125 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 72 72 72 125 125 125 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 70 70 70 123 123 123 F F F 13 260-279 71 71 71 124 124 124 F F F 14 280-299 75 75 75 128 128 128 F F F 15 300-319 69 69 69 120 120 120 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 75 75 75 128 128 128 F F F 18 360-379 75 75 75 128 128 128 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 72 72 72 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 71 71 71 124 124 124 F F F 25 500-519 75 75 75 128 128 128 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 75 75 75 128 128 128 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 F F F Max gen: 75, Max flush: 128 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.681654673Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.726159568Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.727136818Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.727178571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.727194958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.727209526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486} {"msg":"Crucible 453cc248-5871-4615-b710-eb65c9c6e353 has session id: 4f408fc4-4913-4bad-9797-d0d894c3ef4d","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.727714938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Upstairs opts: Upstairs UUID: 453cc248-5871-4615-b710-eb65c9c6e353, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.727762589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Crucible stats registered with UUID: 453cc248-5871-4615-b710-eb65c9c6e353","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.727785625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759129831Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486} {"msg":"453cc248-5871-4615-b710-eb65c9c6e353 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759188551Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759207947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759237892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759263368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759282094Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759298881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759315688Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759332096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759349193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.75936609Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33065)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759614787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57509)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759664358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:53855)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.759699382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.760462729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.760502752Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.760531567Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.760563621Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.760633729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.760721624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761180514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761208539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761290205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761762013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761802646Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761830001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761858096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761903888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761933953Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761958309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.761981045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[0]R flush_numbers[0..12]: [113, 124, 125, 125, 123, 128, 125, 125, 128, 125, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762041514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[0]R generation[0..12]: [65, 71, 72, 72, 70, 75, 72, 72, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762070819Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762086506Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[1]R flush_numbers[0..12]: [113, 124, 125, 125, 123, 128, 125, 125, 128, 125, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762100854Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[1]R generation[0..12]: [65, 71, 72, 72, 70, 75, 72, 72, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762117761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762132278Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[2]R flush_numbers[0..12]: [113, 124, 125, 125, 123, 128, 125, 125, 128, 125, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762146016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[2]R generation[0..12]: [65, 71, 72, 72, 70, 75, 72, 72, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762159553Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762173321Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762186129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762199157Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Next flush: 129","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-09-27T17:12:56.762222493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.76223549Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"":"downstairs","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762248708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762260546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"0","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762273824Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"1","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762286942Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"client":"2","":"downstairs","session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762300429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762314327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"453cc248-5871-4615-b710-eb65c9c6e353 is now active with session: 4f408fc4-4913-4bad-9797-d0d894c3ef4d","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762327244Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762343242Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486,"session_id":"4f408fc4-4913-4bad-9797-d0d894c3ef4d"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:12:56.762356059Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 156 len 36864 data: 5 5 5 6 7 5 6 8 7 02/30 Read block 441 len 28672 03/30 Write block 297 len 28672 data: 2 2 2 2 2 3 4 04/30 Write block 27 len 12288 data: 5 3 5 05/30 Flush 06/30 Write block 361 len 12288 data: 9 12 11 07/30 Write block 340 len 28672 data: 11 10 8 7 8 3 4 08/30 Read block 41 len 12288 09/30 Flush 10/30 Write block 60 len 40960 data: 4 4 5 5 4 5 7 7 7 7 11/30 Write block 169 len 36864 data: 7 6 7 7 5 6 5 4 4 12/30 Read block 103 len 28672 13/30 Write block 162 len 8192 data: 7 9 14/30 Read block 461 len 16384 15/30 Read block 332 len 28672 16/30 Write block 181 len 12288 data: 5 6 6 17/30 Read block 466 len 32768 18/30 Write block 546 len 8192 data: 9 9 19/30 Write block 288 len 40960 data: 7 7 5 5 5 4 4 2 2 3 20/30 Write block 308 len 12288 data: 9 9 9 21/30 Flush 22/30 Write block 112 len 4096 data: 7 23/30 Read block 569 len 16384 24/30 Read block 486 len 8192 25/30 Write block 590 len 40960 data: 7 8 9 8 7 6 5 4 4 3 26/30 Read block 461 len 24576 27/30 Write block 489 len 12288 data: 4 3 4 28/30 Read block 440 len 24576 29/30 Write block 489 len 24576 data: 5 4 5 4 3 4 30/30 Write block 266 len 16384 data: 5 5 4 5 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:12:56.788236575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 76 76 76 129 129 129 F F F 2 040-059 72 72 72 125 125 125 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 70 70 70 123 123 123 F F F 5 100-119 75 75 75 128 128 128 T T T 6 120-139 72 72 72 125 125 125 F F F 7 140-159 76 76 76 129 129 129 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 76 76 76 131 131 131 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 70 70 70 123 123 123 F F F 13 260-279 71 71 71 124 124 124 T T T 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 76 76 76 130 130 130 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 72 72 72 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 71 71 71 124 124 124 T T T 25 500-519 75 75 75 128 128 128 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 76 76 76 131 131 131 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 61 61 61 106 106 106 T T T Max gen: 76, Max flush: 131 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 F F F 1 020-039 76 76 76 129 129 129 F F F 2 040-059 72 72 72 125 125 125 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 70 70 70 123 123 123 F F F 5 100-119 77 77 77 132 132 132 F F F 6 120-139 72 72 72 125 125 125 F F F 7 140-159 76 76 76 129 129 129 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 76 76 76 131 131 131 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 75 75 75 128 128 128 F F F 12 240-259 70 70 70 123 123 123 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 76 76 76 130 130 130 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 68 68 68 119 119 119 F F F 21 420-439 72 72 72 125 125 125 F F F 22 440-459 74 74 74 126 126 126 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 77 77 77 132 132 132 F F F 25 500-519 75 75 75 128 128 128 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 76 76 76 131 131 131 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 77, Max flush: 132 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.188309835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.232800562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.233644316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.233694897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.233713473Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.23373147Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494} {"msg":"Crucible 74dd517a-df6f-4d1b-8679-a937e1900c12 has session id: 9f1dee0e-efb8-4a05-be1c-62667485fa07","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.234190081Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Upstairs opts: Upstairs UUID: 74dd517a-df6f-4d1b-8679-a937e1900c12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.234233523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Crucible stats registered with UUID: 74dd517a-df6f-4d1b-8679-a937e1900c12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.234257449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.264884009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494} {"msg":"74dd517a-df6f-4d1b-8679-a937e1900c12 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.26493827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.264959206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.264988581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265031683Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265065498Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265096182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265120918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265154562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265181117Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265208733Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_connection connected from Ok(127.0.0.1:49008)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265299647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_connection connected from Ok(127.0.0.1:49187)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265350788Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_connection connected from Ok(127.0.0.1:62998)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.265661194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266171905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266216787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266254001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266288305Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266554558Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266594551Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266964277Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.266994902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267144546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267567872Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267595707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267610465Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267709588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267728714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267742362Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.26781079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267830207Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[0]R flush_numbers[0..12]: [113, 129, 125, 131, 123, 132, 125, 129, 131, 131, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267855822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[0]R generation[0..12]: [65, 76, 72, 76, 70, 77, 72, 76, 76, 76, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267870269Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267884917Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[1]R flush_numbers[0..12]: [113, 129, 125, 131, 123, 132, 125, 129, 131, 131, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267898825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[1]R generation[0..12]: [65, 76, 72, 76, 70, 77, 72, 76, 76, 76, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267912662Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.26792669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[2]R flush_numbers[0..12]: [113, 129, 125, 131, 123, 132, 125, 129, 131, 131, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267944707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[2]R generation[0..12]: [65, 76, 72, 76, 70, 77, 72, 76, 76, 76, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267958934Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267972822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267985669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.267998877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268011855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268024603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"":"downstairs","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268037571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:13:01.268049548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"0","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268070015Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"1","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268082943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"client":"2","":"downstairs","session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.2680961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268108358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"74dd517a-df6f-4d1b-8679-a937e1900c12 is now active with session: 9f1dee0e-efb8-4a05-be1c-62667485fa07","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268121176Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268133124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494,"session_id":"9f1dee0e-efb8-4a05-be1c-62667485fa07"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:01.268145272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 413 len 40960 data: 9 8 10 8 8 5 6 5 5 6 02/30 Read block 103 len 4096 03/30 Write block 239 len 32768 data: 7 6 7 6 6 4 5 4 04/30 Flush 05/30 Write block 454 len 20480 data: 8 6 8 8 8 06/30 Read block 194 len 36864 07/30 Read block 358 len 24576 08/30 Read block 475 len 36864 09/30 Write block 439 len 4096 data: 3 10/30 Read block 187 len 24576 11/30 Write block 80 len 20480 data: 8 5 5 7 9 12/30 Flush 13/30 Write block 421 len 20480 data: 6 7 6 6 7 14/30 Read block 0 len 32768 15/30 Write block 53 len 8192 data: 9 9 16/30 Read block 465 len 36864 17/30 Read block 590 len 20480 18/30 Write block 456 len 36864 data: 9 9 9 6 5 5 6 7 8 19/30 Write block 6 len 28672 data: 8 9 8 7 8 9 11 20/30 Write block 341 len 12288 data: 11 9 8 21/30 Write block 61 len 36864 data: 5 6 6 5 6 8 8 8 8 22/30 Read block 206 len 24576 23/30 Write block 116 len 32768 data: 8 9 12 10 8 10 10 10 24/30 Write block 154 len 8192 data: 5 6 25/30 Read block 140 len 4096 26/30 Write block 53 len 36864 data: 10 10 6 5 7 7 5 5 6 27/30 Write block 390 len 4096 data: 5 28/30 Read block 326 len 20480 29/30 Read block 75 len 36864 30/30 Read block 451 len 36864 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 5 Done Done Done false 14 Acked 1013 Read 8 Done Done Done false 15 Acked 1014 Write 2 Done Done Done false 16 Acked 1015 Read 9 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:01.291359682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 113 113 113 T T T 1 020-039 76 76 76 129 129 129 F F F 2 040-059 72 72 72 125 125 125 T T T 3 060-079 76 76 76 131 131 131 T T T 4 080-099 78 78 78 134 134 134 F F F 5 100-119 77 77 77 132 132 132 T T T 6 120-139 72 72 72 125 125 125 T T T 7 140-159 76 76 76 129 129 129 T T T 8 160-179 76 76 76 131 131 131 F F F 9 180-199 76 76 76 131 131 131 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 76 76 76 130 130 130 T T T 18 360-379 76 76 76 130 130 130 F F F 19 380-399 72 72 72 125 125 125 T T T 20 400-419 78 78 78 133 133 133 F F F 21 420-439 78 78 78 134 134 134 T T T 22 440-459 78 78 78 134 134 134 T T T 23 460-479 69 69 69 120 120 120 T T T 24 480-499 77 77 77 132 132 132 F F F 25 500-519 75 75 75 128 128 128 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 76 76 76 131 131 131 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 78, Max flush: 134 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 135 135 135 F F F 1 020-039 76 76 76 129 129 129 F F F 2 040-059 79 79 79 135 135 135 F F F 3 060-079 79 79 79 135 135 135 F F F 4 080-099 78 78 78 134 134 134 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 79 79 79 135 135 135 F F F 7 140-159 79 79 79 135 135 135 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 76 76 76 131 131 131 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 79 79 79 135 135 135 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 79 79 79 135 135 135 F F F 20 400-419 78 78 78 133 133 133 F F F 21 420-439 79 79 79 135 135 135 F F F 22 440-459 79 79 79 135 135 135 F F F 23 460-479 79 79 79 135 135 135 F F F 24 480-499 77 77 77 132 132 132 F F F 25 500-519 75 75 75 128 128 128 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 76 76 76 131 131 131 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 79, Max flush: 135 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.69661337Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:05.739549738Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.740138526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.772321905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.773226208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.773264072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.773278069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.773292327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502} {"msg":"Crucible 5992c2be-b478-4a7c-920c-aa5272f2f70b has session id: 8c2a4985-65b6-4f6a-a711-7ff15c534a64","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.773722202Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Upstairs opts: Upstairs UUID: 5992c2be-b478-4a7c-920c-aa5272f2f70b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.773769673Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Crucible stats registered with UUID: 5992c2be-b478-4a7c-920c-aa5272f2f70b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.773793619Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805149312Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502} {"msg":"5992c2be-b478-4a7c-920c-aa5272f2f70b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805203633Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805222519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805251634Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.80527825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805311194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805341019Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805374013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805407117Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805440142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805471676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_connection connected from Ok(127.0.0.1:47228)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805497751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_connection connected from Ok(127.0.0.1:49575)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805569839Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_connection connected from Ok(127.0.0.1:40466)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.805763435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.806243142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.806271967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.806466783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.806500857Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.806644592Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.806682085Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807099103Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807129608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807421967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807707767Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807732123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.80774658Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807814519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807837564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.807851752Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.80797947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808012604Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[0]R flush_numbers[0..12]: [135, 129, 135, 135, 134, 135, 135, 135, 131, 131, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808028121Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[0]R generation[0..12]: [79, 76, 79, 79, 78, 79, 79, 79, 76, 76, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808042549Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808056886Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[1]R flush_numbers[0..12]: [135, 129, 135, 135, 134, 135, 135, 135, 131, 131, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808070614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[1]R generation[0..12]: [79, 76, 79, 79, 78, 79, 79, 79, 76, 76, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808084281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808098309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[2]R flush_numbers[0..12]: [135, 129, 135, 135, 134, 135, 135, 135, 131, 131, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808115286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[2]R generation[0..12]: [79, 76, 79, 79, 78, 79, 79, 79, 76, 76, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808128964Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808158868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808172846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808186284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808199212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808212169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"":"downstairs","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808225507Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Transition from WaitQuorum to Active","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808248493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"0","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808261741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"1","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808275008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"client":"2","":"downstairs","session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808288186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808300424Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"5992c2be-b478-4a7c-920c-aa5272f2f70b is now active with session: 8c2a4985-65b6-4f6a-a711-7ff15c534a64","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808313172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.80832523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502,"session_id":"8c2a4985-65b6-4f6a-a711-7ff15c534a64"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:05.808339707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 24576 data: 11 11 13 11 12 12 02/30 Write block 430 len 32768 data: 4 5 5 4 6 6 3 3 03/30 Write block 453 len 16384 data: 8 9 7 10 04/30 Read block 295 len 40960 05/30 Write block 28 len 12288 data: 4 6 6 06/30 Write block 136 len 28672 data: 7 8 9 9 7 6 8 07/30 Flush 08/30 Write block 70 len 24576 data: 8 10 11 8 9 9 09/30 Read block 574 len 28672 10/30 Read block 133 len 4096 11/30 Write block 187 len 32768 data: 8 7 6 8 6 6 4 5 12/30 Read block 75 len 40960 13/30 Flush 14/30 Write block 363 len 28672 data: 12 11 9 9 10 8 9 15/30 Flush 16/30 Write block 4 len 20480 data: 5 7 9 10 9 17/30 Read block 224 len 8192 18/30 Write block 247 len 8192 data: 4 6 19/30 Read block 33 len 4096 20/30 Read block 558 len 32768 21/30 Write block 551 len 28672 data: 13 13 9 8 6 6 5 22/30 Read block 552 len 32768 23/30 Read block 512 len 28672 24/30 Flush 25/30 Write block 515 len 16384 data: 12 14 13 10 26/30 Write block 341 len 12288 data: 13 10 9 27/30 Read block 457 len 16384 28/30 Write block 425 len 24576 data: 8 5 5 6 5 5 29/30 Write block 15 len 28672 data: 10 10 10 10 11 7 7 30/30 Read block 299 len 36864 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 7 Sent Done Done false 30 Acked 1029 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 4 6 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:05.834849334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 139 139 139 T T T 1 020-039 80 80 80 136 136 136 T T T 2 040-059 79 79 79 135 135 135 F F F 3 060-079 80 80 80 137 137 137 F F F 4 080-099 78 78 78 134 134 134 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 80 80 80 136 136 136 F F F 7 140-159 80 80 80 136 136 136 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 80 80 80 137 137 137 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 80 80 80 139 139 139 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 80 80 80 136 136 136 F F F 17 340-359 80 80 80 136 136 136 T T T 18 360-379 80 80 80 138 138 138 F F F 19 380-399 79 79 79 135 135 135 F F F 20 400-419 78 78 78 133 133 133 F F F 21 420-439 80 80 80 136 136 136 T T T 22 440-459 80 80 80 136 136 136 F F F 23 460-479 79 79 79 135 135 135 F F F 24 480-499 77 77 77 132 132 132 F F F 25 500-519 75 75 75 128 128 128 T T T 26 520-539 75 75 75 128 128 128 F F F 27 540-559 80 80 80 139 139 139 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 80, Max flush: 139 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 140 140 140 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 79 79 79 135 135 135 F F F 3 060-079 80 80 80 137 137 137 F F F 4 080-099 78 78 78 134 134 134 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 80 80 80 136 136 136 F F F 7 140-159 80 80 80 136 136 136 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 80 80 80 137 137 137 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 80 80 80 139 139 139 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 80 80 80 136 136 136 F F F 17 340-359 81 81 81 140 140 140 F F F 18 360-379 80 80 80 138 138 138 F F F 19 380-399 79 79 79 135 135 135 F F F 20 400-419 78 78 78 133 133 133 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 F F F 23 460-479 79 79 79 135 135 135 F F F 24 480-499 77 77 77 132 132 132 F F F 25 500-519 81 81 81 140 140 140 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 80 80 80 139 139 139 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 81, Max flush: 140 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.171010085Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.214680945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.215481866Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.21551934Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.215534057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.215549045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510} {"msg":"Crucible 095f35fe-6512-4685-b403-d88d1c2b4543 has session id: 16710c4f-58da-431c-81c1-78020195fa15","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.215994107Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Upstairs opts: Upstairs UUID: 095f35fe-6512-4685-b403-d88d1c2b4543, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.216039399Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Crucible stats registered with UUID: 095f35fe-6512-4685-b403-d88d1c2b4543","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.216063695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.246816204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510} {"msg":"095f35fe-6512-4685-b403-d88d1c2b4543 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.246864115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.246883512Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.246912367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.246937192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.246969367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.246999721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.247021048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.247042764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.247072719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.247093185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_connection connected from Ok(127.0.0.1:33604)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.247242909Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_connection connected from Ok(127.0.0.1:46368)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.247301249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_connection connected from Ok(127.0.0.1:46124)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.247333583Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.248141013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.248170648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.248308014Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.248335069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.248546632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.248584836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.248893412Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.249091957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.249408882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.249566855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.249599499Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.249620485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.24965205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.249667087Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.249680555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250000259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250027475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[0]R flush_numbers[0..12]: [140, 140, 135, 137, 134, 135, 136, 136, 131, 137, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250051281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[0]R generation[0..12]: [81, 81, 79, 80, 78, 79, 80, 80, 76, 80, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250066308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250080805Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[1]R flush_numbers[0..12]: [140, 140, 135, 137, 134, 135, 136, 136, 131, 137, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250094623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[1]R generation[0..12]: [81, 81, 79, 80, 78, 79, 80, 80, 76, 80, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250108231Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250122228Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[2]R flush_numbers[0..12]: [140, 140, 135, 137, 134, 135, 136, 136, 131, 137, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250135956Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[2]R generation[0..12]: [81, 81, 79, 80, 78, 79, 80, 80, 76, 80, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250149433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250163381Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250180368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250194196Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250207243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250220221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"":"downstairs","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250233669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250295588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"0","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250324793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"1","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.25033948Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"client":"2","":"downstairs","session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250353288Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250365466Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"095f35fe-6512-4685-b403-d88d1c2b4543 is now active with session: 16710c4f-58da-431c-81c1-78020195fa15","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250378284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.25039951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510,"session_id":"16710c4f-58da-431c-81c1-78020195fa15"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:10.250414117Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 20480 data: 6 6 8 10 7 02/30 Write block 551 len 12288 data: 14 14 10 03/30 Read block 453 len 8192 04/30 Read block 169 len 28672 05/30 Flush 06/30 Write block 461 len 20480 data: 6 7 8 9 7 07/30 Read block 580 len 4096 08/30 Read block 171 len 40960 09/30 Read block 123 len 4096 10/30 Write block 501 len 36864 data: 5 6 5 5 3 3 6 8 8 11/30 Write block 173 len 16384 data: 6 7 6 5 12/30 Write block 130 len 28672 data: 7 7 7 5 7 6 8 13/30 Read block 168 len 16384 14/30 Write block 352 len 12288 data: 4 5 4 15/30 Write block 60 len 12288 data: 6 7 7 16/30 Flush 17/30 Flush 18/30 Write block 164 len 8192 data: 8 6 19/30 Read block 378 len 4096 20/30 Read block 225 len 32768 21/30 Write block 402 len 24576 data: 7 6 7 7 8 8 22/30 Read block 293 len 12288 23/30 Read block 73 len 8192 24/30 Read block 254 len 4096 25/30 Read block 299 len 8192 26/30 Flush 27/30 Write block 95 len 16384 data: 7 9 8 8 28/30 Read block 346 len 24576 29/30 Write block 190 len 12288 data: 9 7 7 30/30 Read block 498 len 36864 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 3 Done Done Sent false 30 Acked 1029 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:10.274341374Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 140 140 140 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 79 79 79 135 135 135 F F F 3 060-079 82 82 82 142 142 142 F F F 4 080-099 82 82 82 141 141 141 T T T 5 100-119 79 79 79 135 135 135 F F F 6 120-139 82 82 82 142 142 142 F F F 7 140-159 80 80 80 136 136 136 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 80 80 80 137 137 137 T T T 10 200-219 75 75 75 128 128 128 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 80 80 80 139 139 139 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 80 80 80 136 136 136 F F F 17 340-359 82 82 82 142 142 142 F F F 18 360-379 80 80 80 138 138 138 F F F 19 380-399 79 79 79 135 135 135 F F F 20 400-419 82 82 82 144 144 144 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 F F F 23 460-479 82 82 82 142 142 142 F F F 24 480-499 77 77 77 132 132 132 F F F 25 500-519 82 82 82 142 142 142 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 82 82 82 141 141 141 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 82, Max flush: 144 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 140 140 140 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 79 79 79 135 135 135 F F F 3 060-079 82 82 82 142 142 142 F F F 4 080-099 83 83 83 145 145 145 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 82 82 82 142 142 142 F F F 7 140-159 80 80 80 136 136 136 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 83 83 83 145 145 145 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 80 80 80 139 139 139 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 80 80 80 136 136 136 F F F 17 340-359 82 82 82 142 142 142 F F F 18 360-379 80 80 80 138 138 138 F F F 19 380-399 79 79 79 135 135 135 F F F 20 400-419 82 82 82 144 144 144 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 F F F 23 460-479 82 82 82 142 142 142 F F F 24 480-499 77 77 77 132 132 132 F F F 25 500-519 82 82 82 142 142 142 F F F 26 520-539 75 75 75 128 128 128 F F F 27 540-559 82 82 82 141 141 141 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 83, Max flush: 145 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.550489738Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.59455376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.595464152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.595502845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.595517292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.59553194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518} {"msg":"Crucible d1e9271c-2c87-4d3e-8da4-bddb65570d13 has session id: 07c818d3-0e01-443e-8aa1-9a7bc6dc98b9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.596013296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Upstairs opts: Upstairs UUID: d1e9271c-2c87-4d3e-8da4-bddb65570d13, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.596061118Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Crucible stats registered with UUID: d1e9271c-2c87-4d3e-8da4-bddb65570d13","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.596088693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628537106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518} {"msg":"d1e9271c-2c87-4d3e-8da4-bddb65570d13 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628600535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.62863375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628679862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628716665Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628751809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628785153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628820807Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628855761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628891705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628926919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47547)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628956504Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56865)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.628984509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50012)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.629013914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.629736918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.629766363Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.629787969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.629802736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.629938043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.629961778Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.630484907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.6305268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.630692641Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631246915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631283819Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631312823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631337059Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631364834Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.63138926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631409797Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631431893Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[0]R flush_numbers[0..12]: [140, 140, 135, 142, 145, 135, 142, 136, 144, 145, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631457068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[0]R generation[0..12]: [81, 81, 79, 82, 83, 79, 82, 80, 82, 83, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631486533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631511109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[1]R flush_numbers[0..12]: [140, 140, 135, 142, 145, 135, 142, 136, 144, 145, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631533795Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[1]R generation[0..12]: [81, 81, 79, 82, 83, 79, 82, 80, 82, 83, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631556581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631581307Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[2]R flush_numbers[0..12]: [140, 140, 135, 142, 145, 135, 142, 136, 144, 145, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631607812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[2]R generation[0..12]: [81, 81, 79, 82, 83, 79, 82, 80, 82, 83, 75, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631633168Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631657563Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631684719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631762355Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.63179266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631816366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"":"downstairs","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631843041Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631861958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"0","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631891373Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"1","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631916498Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"client":"2","":"downstairs","session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631941954Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631970389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"d1e9271c-2c87-4d3e-8da4-bddb65570d13 is now active with session: 07c818d3-0e01-443e-8aa1-9a7bc6dc98b9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.631985417Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.632006873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518,"session_id":"07c818d3-0e01-443e-8aa1-9a7bc6dc98b9"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:14.63202344Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 532 len 16384 data: 7 6 6 8 02/30 Read block 215 len 20480 03/30 Write block 195 len 12288 data: 7 9 9 04/30 Write block 13 len 24576 data: 12 9 11 11 11 11 05/30 Write block 479 len 12288 data: 5 6 6 06/30 Read block 362 len 24576 07/30 Flush 08/30 Write block 229 len 16384 data: 7 7 6 5 09/30 Flush 10/30 Write block 235 len 40960 data: 9 8 8 8 8 7 8 7 7 5 11/30 Write block 550 len 32768 data: 13 15 15 11 9 7 7 6 12/30 Read block 323 len 40960 13/30 Write block 327 len 20480 data: 6 6 5 5 6 14/30 Write block 571 len 36864 data: 7 9 8 8 9 7 7 7 10 15/30 Read block 522 len 40960 16/30 Read block 85 len 28672 17/30 Write block 402 len 12288 data: 8 7 8 18/30 Read block 500 len 32768 19/30 Write block 524 len 28672 data: 10 9 5 4 5 10 10 20/30 Read block 117 len 24576 21/30 Write block 280 len 16384 data: 7 7 8 7 22/30 Write block 221 len 16384 data: 5 6 5 6 23/30 Read block 2 len 36864 24/30 Read block 113 len 8192 25/30 Write block 552 len 28672 data: 16 12 10 8 8 7 5 26/30 Write block 385 len 36864 data: 6 6 6 6 6 6 7 5 6 27/30 Read block 560 len 12288 28/30 Read block 343 len 24576 29/30 Write block 405 len 32768 data: 8 9 9 8 7 8 8 9 30/30 Read block 432 len 24576 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 10 Acked 1009 Write 10 Done Done Done false 11 Acked 1010 Write 8 Done Done Done false 12 Acked 1011 Read 10 Done Done Done false 13 Acked 1012 Write 5 Done Done Done false 14 Acked 1013 Write 9 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Read 7 Done Done Done false 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 8 Done Sent Sent false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 21 19 19 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:14.655936968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 146 146 146 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 79 79 79 135 135 135 F F F 3 060-079 82 82 82 142 142 142 F F F 4 080-099 83 83 83 145 145 145 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 82 82 82 142 142 142 F F F 7 140-159 80 80 80 136 136 136 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 84 84 84 147 147 147 T T T 12 240-259 80 80 80 139 139 139 T T T 13 260-279 77 77 77 132 132 132 F F F 14 280-299 76 76 76 131 131 131 T T T 15 300-319 76 76 76 131 131 131 F F F 16 320-339 80 80 80 136 136 136 T T T 17 340-359 82 82 82 142 142 142 F F F 18 360-379 80 80 80 138 138 138 F F F 19 380-399 79 79 79 135 135 135 T T T 20 400-419 82 82 82 144 144 144 T T T 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 F F F 23 460-479 84 84 84 146 146 146 F F F 24 480-499 84 84 84 146 146 146 F F F 25 500-519 82 82 82 142 142 142 F F F 26 520-539 84 84 84 146 146 146 T T T 27 540-559 82 82 82 141 141 141 T T T 28 560-579 67 67 67 117 117 117 T T T 29 580-599 77 77 77 132 132 132 F F F Max gen: 84, Max flush: 147 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 146 146 146 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 79 79 79 135 135 135 F F F 3 060-079 82 82 82 142 142 142 F F F 4 080-099 83 83 83 145 145 145 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 82 82 82 142 142 142 F F F 7 140-159 80 80 80 136 136 136 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 75 75 75 128 128 128 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 85 85 85 148 148 148 F F F 13 260-279 77 77 77 132 132 132 F F F 14 280-299 85 85 85 148 148 148 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 85 85 85 148 148 148 F F F 17 340-359 82 82 82 142 142 142 F F F 18 360-379 80 80 80 138 138 138 F F F 19 380-399 85 85 85 148 148 148 F F F 20 400-419 85 85 85 148 148 148 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 F F F 23 460-479 84 84 84 146 146 146 F F F 24 480-499 84 84 84 146 146 146 F F F 25 500-519 82 82 82 142 142 142 F F F 26 520-539 85 85 85 148 148 148 F F F 27 540-559 85 85 85 148 148 148 F F F 28 560-579 85 85 85 148 148 148 F F F 29 580-599 77 77 77 132 132 132 F F F Max gen: 85, Max flush: 148 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.033326205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:19.0769109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.07760509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.110365709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.111350447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.111395269Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.111412996Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.111431073Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526} {"msg":"Crucible 3236771f-1ad7-4c80-840c-611fb6227642 has session id: cda5f23f-4d84-40d6-9a2b-8f77e67924ce","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.11185237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Upstairs opts: Upstairs UUID: 3236771f-1ad7-4c80-840c-611fb6227642, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.11190678Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Crucible stats registered with UUID: 3236771f-1ad7-4c80-840c-611fb6227642","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.111942174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143486065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526} {"msg":"3236771f-1ad7-4c80-840c-611fb6227642 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143538836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143557453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143600965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143634879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143667333Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143701508Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143736881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143772015Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143807689Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143842423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:57777)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143879836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:44412)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143920259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:60215)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.143958003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.144710602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.144748235Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.144765082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.144784509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.145038555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.145069979Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.145519681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.145687292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.145713987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.14616078Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146189994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146205922Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146356206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146382311Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146396838Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146409186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146427313Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[0]R flush_numbers[0..12]: [146, 140, 135, 142, 145, 135, 142, 136, 144, 146, 128, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146441671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[0]R generation[0..12]: [84, 81, 79, 82, 83, 79, 82, 80, 82, 84, 75, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146455418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146469536Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[1]R flush_numbers[0..12]: [146, 140, 135, 142, 145, 135, 142, 136, 144, 146, 128, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146483313Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[1]R generation[0..12]: [84, 81, 79, 82, 83, 79, 82, 80, 82, 84, 75, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146496811Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146510589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[2]R flush_numbers[0..12]: [146, 140, 135, 142, 145, 135, 142, 136, 144, 146, 128, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146524096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[2]R generation[0..12]: [84, 81, 79, 82, 83, 79, 82, 80, 82, 84, 75, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146537454Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146551172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146563829Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146577067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146589645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.14661638Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"":"downstairs","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146643576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146659933Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"0","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146682949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"1","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146702825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"client":"2","":"downstairs","session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:13:19.146722402Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146745228Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"3236771f-1ad7-4c80-840c-611fb6227642 is now active with session: cda5f23f-4d84-40d6-9a2b-8f77e67924ce","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146759725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146771863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526,"session_id":"cda5f23f-4d84-40d6-9a2b-8f77e67924ce"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:19.146784081Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 157 len 12288 data: 6 6 7 02/30 Write block 74 len 24576 data: 10 10 10 6 6 9 03/30 Write block 162 len 24576 data: 8 10 9 7 8 7 04/30 Write block 138 len 4096 data: 10 05/30 Read block 373 len 40960 06/30 Write block 271 len 24576 data: 4 4 5 5 5 8 07/30 Read block 375 len 20480 08/30 Read block 399 len 40960 09/30 Write block 385 len 20480 data: 7 7 7 7 7 10/30 Read block 330 len 16384 11/30 Write block 55 len 20480 data: 7 6 8 8 6 12/30 Read block 508 len 12288 13/30 Write block 576 len 4096 data: 8 14/30 Read block 15 len 16384 15/30 Write block 208 len 20480 data: 5 8 7 7 5 16/30 Write block 573 len 36864 data: 9 9 10 9 8 8 11 10 10 17/30 Write block 550 len 20480 data: 14 16 17 13 11 18/30 Write block 94 len 4096 data: 7 19/30 Read block 395 len 28672 20/30 Read block 395 len 12288 21/30 Read block 330 len 4096 22/30 Write block 60 len 4096 data: 7 23/30 Read block 186 len 24576 24/30 Read block 3 len 24576 25/30 Flush 26/30 Write block 557 len 16384 data: 8 6 6 6 27/30 Write block 533 len 20480 data: 7 7 9 7 7 28/30 Read block 573 len 24576 29/30 Write block 328 len 8192 data: 7 6 30/30 Read block 244 len 40960 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:19.168429814Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 146 146 146 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 86 86 86 149 149 149 F F F 4 080-099 86 86 86 149 149 149 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 86 86 86 149 149 149 F F F 8 160-179 86 86 86 149 149 149 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 85 85 85 148 148 148 F F F 13 260-279 86 86 86 149 149 149 F F F 14 280-299 85 85 85 148 148 148 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 85 85 85 148 148 148 T T T 17 340-359 82 82 82 142 142 142 F F F 18 360-379 80 80 80 138 138 138 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 85 85 85 148 148 148 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 F F F 23 460-479 84 84 84 146 146 146 F F F 24 480-499 84 84 84 146 146 146 F F F 25 500-519 82 82 82 142 142 142 F F F 26 520-539 85 85 85 148 148 148 T T T 27 540-559 86 86 86 149 149 149 T T T 28 560-579 86 86 86 149 149 149 T T T 29 580-599 86 86 86 149 149 149 F F F Max gen: 86, Max flush: 149 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 146 146 146 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 86 86 86 149 149 149 F F F 4 080-099 86 86 86 149 149 149 F F F 5 100-119 79 79 79 135 135 135 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 86 86 86 149 149 149 F F F 8 160-179 86 86 86 149 149 149 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 85 85 85 148 148 148 F F F 13 260-279 86 86 86 149 149 149 F F F 14 280-299 85 85 85 148 148 148 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 87 87 87 150 150 150 F F F 17 340-359 82 82 82 142 142 142 F F F 18 360-379 80 80 80 138 138 138 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 85 85 85 148 148 148 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 F F F 23 460-479 84 84 84 146 146 146 F F F 24 480-499 84 84 84 146 146 146 F F F 25 500-519 82 82 82 142 142 142 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 87 87 87 150 150 150 F F F 29 580-599 86 86 86 149 149 149 F F F Max gen: 87, Max flush: 150 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.511790103Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.556773205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.557771241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.557809414Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.557823642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.55783784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534} {"msg":"Crucible 6fe7d73e-77d1-4c73-8e18-b4c5383679d6 has session id: 52d72d0e-f5ce-4648-968c-2b926f2a214b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.558311097Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Upstairs opts: Upstairs UUID: 6fe7d73e-77d1-4c73-8e18-b4c5383679d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.558355379Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Crucible stats registered with UUID: 6fe7d73e-77d1-4c73-8e18-b4c5383679d6","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.558379065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.589798338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534} {"msg":"6fe7d73e-77d1-4c73-8e18-b4c5383679d6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.589863067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.589896471Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.589941213Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.589984155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590019629Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590049024Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590079669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590111823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590145937Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590182291Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33565)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590219505Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64029)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590256738Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39216)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590288962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.590987841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.591016186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.591031283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.591044971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.591293088Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.5913372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.591893593Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.591922968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.591939065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592507227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592536521Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592553748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592612278Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592640483Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592655451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592667908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592681396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[0]R flush_numbers[0..12]: [146, 140, 149, 149, 149, 135, 149, 149, 149, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592709501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[0]R generation[0..12]: [84, 81, 86, 86, 86, 79, 86, 86, 86, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592737506Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592751714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[1]R flush_numbers[0..12]: [146, 140, 149, 149, 149, 135, 149, 149, 149, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592769171Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[1]R generation[0..12]: [84, 81, 86, 86, 86, 79, 86, 86, 86, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592783048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592797016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[2]R flush_numbers[0..12]: [146, 140, 149, 149, 149, 135, 149, 149, 149, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592810843Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[2]R generation[0..12]: [84, 81, 86, 86, 86, 79, 86, 86, 86, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592824311Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592837969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592850677Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592863714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592876682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.59288953Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"":"downstairs","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592902618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.592914636Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"0","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-09-27T17:13:23.592927693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"1","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.593022027Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"client":"2","":"downstairs","session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.593046373Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.5930629Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"6fe7d73e-77d1-4c73-8e18-b4c5383679d6 is now active with session: 52d72d0e-f5ce-4648-968c-2b926f2a214b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.593078857Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.593100713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534,"session_id":"52d72d0e-f5ce-4648-968c-2b926f2a214b"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:23.593124759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 283 len 12288 data: 8 5 6 02/30 Read block 61 len 40960 03/30 Flush 04/30 Write block 120 len 8192 data: 9 11 05/30 Write block 320 len 36864 data: 5 4 3 3 5 5 5 7 8 06/30 Read block 390 len 40960 07/30 Read block 159 len 40960 08/30 Read block 563 len 12288 09/30 Write block 164 len 4096 data: 10 10/30 Write block 124 len 4096 data: 11 11/30 Read block 322 len 40960 12/30 Write block 286 len 28672 data: 7 7 8 8 6 6 6 13/30 Read block 258 len 8192 14/30 Read block 228 len 20480 15/30 Write block 274 len 8192 data: 6 6 16/30 Read block 302 len 36864 17/30 Read block 221 len 40960 18/30 Read block 312 len 32768 19/30 Read block 329 len 24576 20/30 Read block 137 len 28672 21/30 Read block 35 len 12288 22/30 Write block 100 len 28672 data: 7 7 7 8 6 5 8 23/30 Write block 374 len 8192 data: 11 11 24/30 Write block 450 len 8192 data: 10 8 25/30 Read block 510 len 8192 26/30 Read block 12 len 24576 27/30 Write block 505 len 4096 data: 4 28/30 Read block 313 len 4096 29/30 Read block 78 len 40960 30/30 Write block 248 len 16384 data: 7 6 6 6 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 4 Acked 1003 Write 2 Done Done Done false 5 Acked 1004 Write 9 Done Done Done false 6 Acked 1005 Read 10 Done Done Done false 7 Acked 1006 Read 10 Done Done Done false 8 Acked 1007 Read 3 Done Done Done false 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Write 1 Done Done Done false 11 Acked 1010 Read 10 Done Done Done false 12 Acked 1011 Write 7 Done Done Done false 13 Acked 1012 Read 2 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Write 2 Done Done Done false 16 Acked 1015 Read 9 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1002 1002 1002 Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:23.614515267Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 146 146 146 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 86 86 86 149 149 149 F F F 4 080-099 86 86 86 149 149 149 F F F 5 100-119 79 79 79 135 135 135 T T T 6 120-139 86 86 86 149 149 149 T T T 7 140-159 86 86 86 149 149 149 F F F 8 160-179 86 86 86 149 149 149 T T T 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 85 85 85 148 148 148 T T T 13 260-279 86 86 86 149 149 149 T T T 14 280-299 88 88 88 151 151 151 T T T 15 300-319 76 76 76 131 131 131 F F F 16 320-339 87 87 87 150 150 150 T T T 17 340-359 82 82 82 142 142 142 F F F 18 360-379 80 80 80 138 138 138 T T T 19 380-399 86 86 86 149 149 149 F F F 20 400-419 85 85 85 148 148 148 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 80 80 80 136 136 136 T T T 23 460-479 84 84 84 146 146 146 F F F 24 480-499 84 84 84 146 146 146 F F F 25 500-519 82 82 82 142 142 142 T T T 26 520-539 87 87 87 150 150 150 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 87 87 87 150 150 150 F F F 29 580-599 86 86 86 149 149 149 F F F Max gen: 88, Max flush: 151 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 146 146 146 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 86 86 86 149 149 149 F F F 4 080-099 86 86 86 149 149 149 F F F 5 100-119 89 89 89 152 152 152 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 149 149 149 F F F 8 160-179 89 89 89 152 152 152 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 89 89 89 152 152 152 F F F 13 260-279 89 89 89 152 152 152 F F F 14 280-299 89 89 89 152 152 152 F F F 15 300-319 76 76 76 131 131 131 F F F 16 320-339 89 89 89 152 152 152 F F F 17 340-359 82 82 82 142 142 142 F F F 18 360-379 89 89 89 152 152 152 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 85 85 85 148 148 148 F F F 21 420-439 81 81 81 140 140 140 F F F 22 440-459 89 89 89 152 152 152 F F F 23 460-479 84 84 84 146 146 146 F F F 24 480-499 84 84 84 146 146 146 F F F 25 500-519 89 89 89 152 152 152 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 87 87 87 150 150 150 F F F 29 580-599 86 86 86 149 149 149 F F F Max gen: 89, Max flush: 152 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.004412726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:28.046500582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.046939596Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.078789323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.079561819Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.079601822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.079616649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.079631067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542} {"msg":"Crucible d145dc31-2f76-458d-a525-8aa4c6552862 has session id: 45f5ee5c-a7bf-4af9-ae95-2f140a93467f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.080238021Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Upstairs opts: Upstairs UUID: d145dc31-2f76-458d-a525-8aa4c6552862, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.080285973Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Crucible stats registered with UUID: d145dc31-2f76-458d-a525-8aa4c6552862","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.08030392Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111239246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542} {"msg":"d145dc31-2f76-458d-a525-8aa4c6552862 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111301026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111321332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111360855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111395869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111429623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111461368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111495102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111526216Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111559141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111590985Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35852)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.11162326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62382)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111654354Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49892)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.111685139Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.112588122Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.112635163Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.112721239Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.112745504Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.112760592Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.112774309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.113360147Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.113546695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.11357722Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114026572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114066434Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114088501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114108047Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114122205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114135613Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114211289Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114231246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[0]R flush_numbers[0..12]: [146, 140, 149, 149, 149, 152, 152, 149, 152, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114252992Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[0]R generation[0..12]: [84, 81, 86, 86, 86, 89, 89, 86, 89, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114270659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114285087Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[1]R flush_numbers[0..12]: [146, 140, 149, 149, 149, 152, 152, 149, 152, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114298754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[1]R generation[0..12]: [84, 81, 86, 86, 86, 89, 89, 86, 89, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114312542Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114339807Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[2]R flush_numbers[0..12]: [146, 140, 149, 149, 149, 152, 152, 149, 152, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114355824Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[2]R generation[0..12]: [84, 81, 86, 86, 86, 89, 89, 86, 89, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114369432Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114383359Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114396267Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114409305Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114421983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.11443481Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"":"downstairs","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114447788Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114459666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1",Wait for a query_work_queue command to finish before sending IO "pid":1542,"client":"0","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114523015Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"1","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114539442Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"client":"2","":"downstairs","session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.11455275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114564968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"d145dc31-2f76-458d-a525-8aa4c6552862 is now active with session: 45f5ee5c-a7bf-4af9-ae95-2f140a93467f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114577725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114589614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542,"session_id":"45f5ee5c-a7bf-4af9-ae95-2f140a93467f"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:28.114601771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 4096 data: 3 02/30 Flush 03/30 Write block 430 len 32768 data: 6 6 6 5 7 7 4 4 04/30 Flush 05/30 Write block 472 len 4096 data: 5 06/30 Write block 475 len 4096 data: 8 07/30 Read block 171 len 20480 08/30 Read block 177 len 28672 09/30 Write block 150 len 12288 data: 6 6 6 10/30 Write block 471 len 40960 data: 5 6 4 6 9 7 5 5 6 7 11/30 Write block 82 len 36864 data: 7 9 11 8 7 7 5 5 5 12/30 Write block 447 len 40960 data: 8 7 9 11 9 7 9 10 8 11 13/30 Read block 336 len 8192 14/30 Write block 445 len 16384 data: 5 8 9 8 15/30 Write block 331 len 28672 data: 7 7 7 8 13 12 12 16/30 Read block 55 len 36864 17/30 Write block 590 len 40960 data: 8 9 10 9 8 7 6 5 5 4 18/30 Read block 570 len 16384 19/30 Write block 307 len 24576 data: 10 10 10 10 9 8 20/30 Read block 129 len 36864 21/30 Read block 358 len 36864 22/30 Write block 81 len 36864 data: 7 8 10 12 9 8 8 6 6 23/30 Read block 382 len 28672 24/30 Read block 370 len 4096 25/30 Write block 400 len 36864 data: 6 6 9 8 9 9 10 10 9 26/30 Read block 128 len 12288 27/30 Read block 25 len 12288 28/30 Read block 110 len 4096 29/30 Read block 320 len 28672 30/30 Read block 427 len 32768 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 5 Acked 1004 Write 1 Done Done Done false 6 Acked 1005 Write 1 Done Done Done false 7 Acked 1006 Read 5 Done Done Done false 8 Acked 1007 Read 7 Done Done Done false 9 Acked 1008 Write 3 Done Done Done false 10 Acked 1009 Write 10 Done Done Done false 11 Acked 1010 Write 9 Done Done Done false 12 Acked 1011 Write 10 Done Done Done false 13 Acked 1012 Read 2 Done Done Done false 14 Acked 1013 Write 4 Done Done Done false 15 Acked 1014 Write 7 Done Done Done false 16 Acked 1015 Read 9 Done Done Done false 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Read 4 Done Done Done false 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Read 9 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 26 25 26 77 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:28.137789716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 153 153 153 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 86 86 86 149 149 149 F F F 4 080-099 86 86 86 149 149 149 T T T 5 100-119 89 89 89 152 152 152 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 149 149 149 T T T 8 160-179 89 89 89 152 152 152 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 89 89 89 152 152 152 F F F 13 260-279 89 89 89 152 152 152 F F F 14 280-299 89 89 89 152 152 152 F F F 15 300-319 76 76 76 131 131 131 T T T 16 320-339 89 89 89 152 152 152 T T T 17 340-359 82 82 82 142 142 142 F F F 18 360-379 89 89 89 152 152 152 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 85 85 85 148 148 148 T T T 21 420-439 90 90 90 154 154 154 F F F 22 440-459 89 89 89 152 152 152 T T T 23 460-479 84 84 84 146 146 146 T T T 24 480-499 84 84 84 146 146 146 T T T 25 500-519 89 89 89 152 152 152 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 87 87 87 150 150 150 F F F 29 580-599 86 86 86 149 149 149 T T T Max gen: 90, Max flush: 154 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 153 153 153 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 86 86 86 149 149 149 F F F 4 080-099 91 91 91 155 155 155 F F F 5 100-119 89 89 89 152 152 152 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 91 91 91 155 155 155 F F F 8 160-179 89 89 89 152 152 152 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 89 89 89 152 152 152 F F F 13 260-279 89 89 89 152 152 152 F F F 14 280-299 89 89 89 152 152 152 F F F 15 300-319 91 91 91 155 155 155 F F F 16 320-339 91 91 91 155 155 155 F F F 17 340-359 82 82 82 142 142 142 F F F 18 360-379 89 89 89 152 152 152 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 91 91 91 155 155 155 F F F 21 420-439 90 90 90 154 154 154 F F F 22 440-459 91 91 91 155 155 155 F F F 23 460-479 91 91 91 155 155 155 F F F 24 480-499 91 91 91 155 155 155 F F F 25 500-519 89 89 89 152 152 152 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 87 87 87 150 150 150 F F F 29 580-599 91 91 91 155 155 155 F F F Max gen: 91, Max flush: 155 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.543992269Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.589567568Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.590560335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.590605087Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.590622184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.590640371Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550} {"msg":"Crucible 858d436c-ee3d-4397-b9c6-1235100e2b50 has session id: 08e26845-23fd-4714-8d01-cb32d29e6326","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.591147373Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Upstairs opts: Upstairs UUID: 858d436c-ee3d-4397-b9c6-1235100e2b50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.591194705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Crucible stats registered with UUID: 858d436c-ee3d-4397-b9c6-1235100e2b50","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.59122102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.622898298Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550} {"msg":"858d436c-ee3d-4397-b9c6-1235100e2b50 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.622948169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.622966606Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623008409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623028845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623046802Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623063089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623080396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623110011Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623138766Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623166581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_connection connected from Ok(127.0.0.1:57682)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623239678Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_connection connected from Ok(127.0.0.1:35809)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623543326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_connection connected from Ok(127.0.0.1:37406)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.623595497Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.624172736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.624200401Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.62438124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.624402156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.624415924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.624429431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.624949911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.624981205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625154405Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625564874Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625591229Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625606227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625664366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625684013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625697611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625753051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625772538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[0]R flush_numbers[0..12]: [153, 140, 149, 149, 155, 152, 152, 155, 152, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625796083Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[0]R generation[0..12]: [90, 81, 86, 86, 91, 89, 89, 91, 89, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.62581421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625828748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[1]R flush_numbers[0..12]: [153, 140, 149, 149, 155, 152, 152, 155, 152, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625842566Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[1]R generation[0..12]: [90, 81, 86, 86, 91, 89, 89, 91, 89, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625855863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625869661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[2]R flush_numbers[0..12]: [153, 140, 149, 149, 155, 152, 152, 155, 152, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625883348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[2]R generation[0..12]: [90, 81, 86, 86, 91, 89, 89, 91, 89, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625896766Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625910623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625923631Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625936779Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625949587Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625962825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"":"downstairs","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.625976242Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.62598907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"0","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-09-27T17:13:32.626004357Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"1","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.626029783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"client":"2","":"downstairs","session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.62604416Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.626056458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"858d436c-ee3d-4397-b9c6-1235100e2b50 is now active with session: 08e26845-23fd-4714-8d01-cb32d29e6326","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.626069376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.626081404Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550,"session_id":"08e26845-23fd-4714-8d01-cb32d29e6326"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:32.626093622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 284 len 4096 data: 6 02/30 Write block 386 len 8192 data: 8 8 03/30 Write block 102 len 28672 data: 8 9 7 6 9 7 7 04/30 Write block 259 len 32768 data: 7 8 7 6 6 5 5 6 05/30 Flush 06/30 Flush 07/30 Write block 451 len 20480 data: 10 8 10 11 9 08/30 Read block 589 len 36864 09/30 Read block 535 len 36864 10/30 Read block 232 len 12288 11/30 Read block 361 len 12288 12/30 Read block 566 len 16384 13/30 Write block 460 len 24576 data: 6 7 8 9 10 8 14/30 Read block 30 len 36864 15/30 Write block 381 len 16384 data: 7 7 9 7 16/30 Write block 69 len 16384 data: 9 9 11 12 17/30 Write block 176 len 16384 data: 6 5 4 3 18/30 Write block 556 len 36864 data: 9 9 7 7 7 8 7 5 6 19/30 Read block 185 len 40960 20/30 Flush 21/30 Write block 3 len 20480 data: 3 6 8 10 11 22/30 Read block 346 len 40960 23/30 Write block 359 len 16384 data: 7 7 10 13 24/30 Read block 56 len 8192 25/30 Write block 375 len 40960 data: 12 8 6 6 7 6 8 8 10 8 26/30 Write block 463 len 40960 data: 10 11 9 7 6 7 6 4 6 7 27/30 Read block 66 len 16384 28/30 Write block 94 len 40960 data: 8 8 10 9 9 9 8 8 9 10 29/30 Read block 590 len 24576 30/30 Read block 96 len 12288 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:32.652356641Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 153 153 153 T T T 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 92 92 92 158 158 158 F F F 4 080-099 91 91 91 155 155 155 T T T 5 100-119 92 92 92 156 156 156 T T T 6 120-139 89 89 89 152 152 152 F F F 7 140-159 91 91 91 155 155 155 F F F 8 160-179 92 92 92 158 158 158 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 91 91 91 155 155 155 F F F 16 320-339 91 91 91 155 155 155 F F F 17 340-359 82 82 82 142 142 142 T T T 18 360-379 89 89 89 152 152 152 T T T 19 380-399 92 92 92 158 158 158 T T T 20 400-419 91 91 91 155 155 155 F F F 21 420-439 90 90 90 154 154 154 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 92 92 92 158 158 158 T T T 24 480-499 91 91 91 155 155 155 F F F 25 500-519 89 89 89 152 152 152 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 92 92 92 158 158 158 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 91 91 91 155 155 155 F F F Max gen: 92, Max flush: 158 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 159 159 159 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 92 92 92 158 158 158 F F F 4 080-099 93 93 93 159 159 159 F F F 5 100-119 93 93 93 159 159 159 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 91 91 91 155 155 155 F F F 8 160-179 92 92 92 158 158 158 F F F 9 180-199 84 84 84 146 146 146 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 85 85 85 148 148 148 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 91 91 91 155 155 155 F F F 16 320-339 91 91 91 155 155 155 F F F 17 340-359 93 93 93 159 159 159 F F F 18 360-379 93 93 93 159 159 159 F F F 19 380-399 93 93 93 159 159 159 F F F 20 400-419 91 91 91 155 155 155 F F F 21 420-439 90 90 90 154 154 154 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 93 93 93 159 159 159 F F F 24 480-499 91 91 91 155 155 155 F F F 25 500-519 89 89 89 152 152 152 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 92 92 92 158 158 158 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 91 91 91 155 155 155 F F F Max gen: 93, Max flush: 159 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.007146731Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.050986242Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.05197438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.052026861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.052054036Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.052079461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558} {"msg":"Crucible 532aee9a-7152-41a1-9341-3941b100c0ac has session id: 91327bea-cc60-4f3b-bbc6-e80f11ccd51a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.052461995Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Upstairs opts: Upstairs UUID: 532aee9a-7152-41a1-9341-3941b100c0ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.052509837Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Crucible stats registered with UUID: 532aee9a-7152-41a1-9341-3941b100c0ac","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.052536472Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083312286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558} {"msg":"532aee9a-7152-41a1-9341-3941b100c0ac active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083413109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083446493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083483037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.08352301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083546366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083568472Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083589888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083611025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083634321Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083656817Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61496)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083946896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47426)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.083986919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55584)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.084015874Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.084443809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.084476864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.084537483Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.084573577Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.084682498Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.084715972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.085274215Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.08530542Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.08536076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.085902956Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.08593207Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.085947218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.085960166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.085973983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.085987241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.085999279Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086012567Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[0]R flush_numbers[0..12]: [159, 140, 149, 158, 159, 159, 152, 155, 158, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086026114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[0]R generation[0..12]: [93, 81, 86, 92, 93, 93, 89, 91, 92, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086039952Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086054009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[1]R flush_numbers[0..12]: [159, 140, 149, 158, 159, 159, 152, 155, 158, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086071266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[1]R generation[0..12]: [93, 81, 86, 92, 93, 93, 89, 91, 92, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086085254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086099141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[2]R flush_numbers[0..12]: [159, 140, 149, 158, 159, 159, 152, 155, 158, 146, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086114189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[2]R generation[0..12]: [93, 81, 86, 92, 93, 93, 89, 91, 92, 84, 86, 85]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086127656Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086157571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086172199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086185426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086198264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086211162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"":"downstairs","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.08622415Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"Wait for a query_work_queue command to finish before sending IO session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086247846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"0","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086261043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"1","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086274051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"client":"2","":"downstairs","session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086287259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086302516Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"532aee9a-7152-41a1-9341-3941b100c0ac is now active with session: 91327bea-cc60-4f3b-bbc6-e80f11ccd51a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086315774Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086328022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558,"session_id":"91327bea-cc60-4f3b-bbc6-e80f11ccd51a"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:37.086340319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 205 len 4096 data: 4 02/30 Read block 394 len 8192 03/30 Read block 569 len 36864 04/30 Read block 36 len 12288 05/30 Write block 318 len 4096 data: 5 06/30 Write block 506 len 36864 data: 4 7 9 9 7 8 9 9 9 07/30 Write block 219 len 12288 data: 8 6 6 08/30 Write block 544 len 4096 data: 8 09/30 Write block 381 len 8192 data: 9 9 10/30 Write block 390 len 40960 data: 7 8 6 7 5 4 4 5 5 5 11/30 Read block 580 len 40960 12/30 Read block 455 len 28672 13/30 Read block 576 len 20480 14/30 Read block 390 len 24576 15/30 Read block 290 len 4096 16/30 Read block 172 len 4096 17/30 Read block 102 len 12288 18/30 Read block 400 len 32768 19/30 Read block 212 len 28672 20/30 Flush 21/30 Write block 172 len 36864 data: 8 7 8 7 7 6 5 4 6 22/30 Write block 159 len 40960 data: 8 8 6 9 11 11 8 9 8 9 23/30 Read block 498 len 16384 24/30 Read block 349 len 32768 25/30 Read block 537 len 12288 26/30 Write block 10 len 40960 data: 9 10 12 13 10 12 12 12 12 12 27/30 Write block 593 len 8192 data: 10 9 28/30 Read block 399 len 8192 29/30 Write block 435 len 20480 data: 8 5 5 3 4 30/30 Write block 404 len 28672 data: 10 10 11 11 10 8 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 5 Sent Sent Sent false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:37.10771622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 159 159 159 T T T 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 92 92 92 158 158 158 F F F 4 080-099 93 93 93 159 159 159 F F F 5 100-119 93 93 93 159 159 159 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 91 91 91 155 155 155 T T T 8 160-179 92 92 92 158 158 158 T T T 9 180-199 84 84 84 146 146 146 T T T 10 200-219 94 94 94 160 160 160 F F F 11 220-239 94 94 94 160 160 160 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 94 94 94 160 160 160 F F F 16 320-339 91 91 91 155 155 155 F F F 17 340-359 93 93 93 159 159 159 F F F 18 360-379 93 93 93 159 159 159 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 91 91 91 155 155 155 T T T 21 420-439 90 90 90 154 154 154 T T T 22 440-459 92 92 92 158 158 158 F F F 23 460-479 93 93 93 159 159 159 F F F 24 480-499 91 91 91 155 155 155 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 91 91 91 155 155 155 T T T Max gen: 94, Max flush: 160 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 92 92 92 158 158 158 F F F 4 080-099 93 93 93 159 159 159 F F F 5 100-119 93 93 93 159 159 159 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 95 95 95 161 161 161 F F F 9 180-199 95 95 95 161 161 161 F F F 10 200-219 94 94 94 160 160 160 F F F 11 220-239 94 94 94 160 160 160 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 94 94 94 160 160 160 F F F 16 320-339 91 91 91 155 155 155 F F F 17 340-359 93 93 93 159 159 159 F F F 18 360-379 93 93 93 159 159 159 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 95 95 95 161 161 161 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 93 93 93 159 159 159 F F F 24 480-499 91 91 91 155 155 155 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 95 95 95 161 161 161 F F F Max gen: 95, Max flush: 161 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.455275568Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.499171689Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.500030469Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.500068253Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.50008244Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.500096998Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566} {"msg":"Crucible f0463731-1c8a-4f88-af7f-a4fad611a64b has session id: 01b968b7-e081-4204-ae7f-3654c4c91287","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.500630855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Upstairs opts: Upstairs UUID: f0463731-1c8a-4f88-af7f-a4fad611a64b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.500677887Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Crucible stats registered with UUID: f0463731-1c8a-4f88-af7f-a4fad611a64b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.500702072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531203515Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566} {"msg":"f0463731-1c8a-4f88-af7f-a4fad611a64b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531248727Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531279451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531324004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531357708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531392552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531426606Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531468709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531503283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531540436Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531572691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_connection connected from Ok(127.0.0.1:39706)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531601985Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_connection connected from Ok(127.0.0.1:57405)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.53162981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_connection connected from Ok(127.0.0.1:61982)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.531658776Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.532340317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.532373341Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.532389858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.532404256Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.532540792Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.532583295Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533080788Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533109193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533231382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533656938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533684323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.53369906Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533738954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.53375768Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533771308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533830298Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533850814Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[0]R flush_numbers[0..12]: [161, 140, 149, 158, 159, 159, 152, 161, 161, 161, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.53387169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[0]R generation[0..12]: [95, 81, 86, 92, 93, 93, 89, 95, 95, 95, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533885708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533899715Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[1]R flush_numbers[0..12]: [161, 140, 149, 158, 159, 159, 152, 161, 161, 161, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533913393Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[1]R generation[0..12]: [95, 81, 86, 92, 93, 93, 89, 95, 95, 95, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533927041Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533940998Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[2]R flush_numbers[0..12]: [161, 140, 149, 158, 159, 159, 152, 161, 161, 161, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533958495Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[2]R generation[0..12]: [95, 81, 86, 92, 93, 93, 89, 95, 95, 95, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533972303Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.53398637Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.533999398Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534012326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534025004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534037642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"":"downstairs","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534050779Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534062657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"0","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534076835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"1","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Transition from WaitQuorum to Active",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.53410504Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"client":"2","":"downstairs","session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534118228Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534130335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"f0463731-1c8a-4f88-af7f-a4fad611a64b is now active with session: 01b968b7-e081-4204-ae7f-3654c4c91287","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534142903Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534154951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566,"session_id":"01b968b7-e081-4204-ae7f-3654c4c91287"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:41.534169559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 4096 data: 11 02/30 Flush 03/30 Write block 230 len 8192 data: 8 7 04/30 Read block 113 len 16384 05/30 Flush 06/30 Write block 98 len 40960 data: 10 10 9 9 10 11 8 7 10 8 07/30 Read block 400 len 8192 08/30 Read block 98 len 32768 09/30 Read block 171 len 36864 10/30 Read block 322 len 20480 11/30 Write block 177 len 16384 data: 7 6 5 7 12/30 Read block 47 len 16384 13/30 Write block 430 len 40960 data: 7 7 7 6 8 9 6 6 4 5 14/30 Read block 379 len 12288 15/30 Read block 332 len 28672 16/30 Write block 383 len 16384 data: 11 9 8 9 17/30 Write block 74 len 8192 data: 11 11 18/30 Read block 339 len 20480 19/30 Flush 20/30 Write block 79 len 16384 data: 10 9 8 9 21/30 Read block 209 len 24576 22/30 Write block 119 len 4096 data: 11 23/30 Write block 342 len 36864 data: 11 10 9 4 5 3 4 4 5 24/30 Read block 55 len 16384 25/30 Flush 26/30 Write block 215 len 20480 data: 5 7 7 7 9 27/30 Read block 96 len 4096 28/30 Write block 197 len 24576 data: 10 10 10 10 9 9 29/30 Write block 359 len 28672 data: 8 8 11 14 13 12 10 30/30 Read block 246 len 28672 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 7 Sent Done Done false 30 Acked 1029 Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 3 5 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:41.560991371Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 96 96 96 165 165 165 F F F 4 080-099 96 96 96 165 165 165 F F F 5 100-119 96 96 96 165 165 165 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 96 96 96 164 164 164 T T T 10 200-219 94 94 94 160 160 160 T T T 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 94 94 94 160 160 160 F F F 16 320-339 91 91 91 155 155 155 F F F 17 340-359 96 96 96 165 165 165 T T T 18 360-379 93 93 93 159 159 159 T T T 19 380-399 96 96 96 164 164 164 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 93 93 93 159 159 159 F F F 24 480-499 91 91 91 155 155 155 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 95 95 95 161 161 161 F F F Max gen: 96, Max flush: 165 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 81 81 81 140 140 140 F F F 2 040-059 86 86 86 149 149 149 F F F 3 060-079 96 96 96 165 165 165 F F F 4 080-099 96 96 96 165 165 165 F F F 5 100-119 96 96 96 165 165 165 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 97 97 97 166 166 166 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 94 94 94 160 160 160 F F F 16 320-339 91 91 91 155 155 155 F F F 17 340-359 97 97 97 166 166 166 F F F 18 360-379 97 97 97 166 166 166 F F F 19 380-399 96 96 96 164 164 164 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 93 93 93 159 159 159 F F F 24 480-499 91 91 91 155 155 155 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 95 95 95 161 161 161 F F F Max gen: 97, Max flush: 166 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.876258718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.920329199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.921303579Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.921350861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.921369758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.921448814Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574} {"msg":"Crucible 4d28cae3-fb35-46a0-9f25-65e4c844ac31 has session id: dc13414c-08fc-4a8d-98da-0784d7735a5f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.921903315Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Upstairs opts: Upstairs UUID: 4d28cae3-fb35-46a0-9f25-65e4c844ac31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.921953076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Crucible stats registered with UUID: 4d28cae3-fb35-46a0-9f25-65e4c844ac31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.921971983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952736369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574} {"msg":"4d28cae3-fb35-46a0-9f25-65e4c844ac31 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952784581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952800958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952824534Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952839171Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952853099Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952866507Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952879804Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952893052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.95290734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.952930955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34374)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.953066562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42437)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.953123142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50700)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.953159016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954001959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954034454Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954049951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954063289Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954201544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.95422486Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954768626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954844333Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.954908461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955418473Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955449407Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955466074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.95548869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955514346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955529143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955541471Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955554799Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[0]R flush_numbers[0..12]: [161, 140, 149, 165, 165, 165, 152, 161, 164, 166, 166, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955568707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[0]R generation[0..12]: [95, 81, 86, 96, 96, 96, 89, 95, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955582484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955596462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[1]R flush_numbers[0..12]: [161, 140, 149, 165, 165, 165, 152, 161, 164, 166, 166, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955610129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[1]R generation[0..12]: [95, 81, 86, 96, 96, 96, 89, 95, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955623547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955637435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[2]R flush_numbers[0..12]: [161, 140, 149, 165, 165, 165, 152, 161, 164, 166, 166, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955651042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[2]R generation[0..12]: [95, 81, 86, 96, 96, 96, 89, 95, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.9556642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955677877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955690605Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955705763Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.95571873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955731598Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"":"downstairs","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955744726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955756644Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"0","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5fWait for a query_work_queue command to finish before sending IO "} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955784109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"1","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955809505Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"client":"2","":"downstairs","session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955826192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955838889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"4d28cae3-fb35-46a0-9f25-65e4c844ac31 is now active with session: dc13414c-08fc-4a8d-98da-0784d7735a5f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955856576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"Set Active after no reconciliation","v":0,"name"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"crucible","level":30,"time":"2024-09-27T17:13:45.955882722Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574,"session_id":"dc13414c-08fc-4a8d-98da-0784d7735a5f"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:45.955897159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 478 len 24576 data: 6 7 8 7 5 7 02/30 Read block 32 len 36864 03/30 Write block 37 len 20480 data: 5 5 6 5 7 04/30 Read block 409 len 20480 05/30 Write block 349 len 28672 data: 5 6 4 5 6 5 5 06/30 Write block 85 len 20480 data: 10 9 9 7 7 07/30 Read block 55 len 32768 08/30 Write block 218 len 8192 data: 8 10 09/30 Read block 359 len 20480 10/30 Write block 427 len 28672 data: 6 7 6 8 8 8 7 11/30 Flush 12/30 Write block 365 len 8192 data: 11 10 13/30 Flush 14/30 Write block 553 len 20480 data: 14 12 9 10 10 15/30 Write block 498 len 36864 data: 7 7 5 6 7 6 6 5 5 16/30 Read block 473 len 40960 17/30 Flush 18/30 Write block 339 len 40960 data: 12 13 14 12 11 10 5 6 4 5 19/30 Read block 387 len 32768 20/30 Read block 445 len 28672 21/30 Read block 524 len 20480 22/30 Flush 23/30 Write block 432 len 12288 data: 9 8 9 24/30 Read block 126 len 16384 25/30 Read block 81 len 40960 26/30 Read block 385 len 36864 27/30 Read block 245 len 24576 28/30 Read block 125 len 36864 29/30 Read block 521 len 36864 30/30 Read block 475 len 32768 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:45.984520208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 98 98 98 167 167 167 F F F 2 040-059 98 98 98 167 167 167 F F F 3 060-079 96 96 96 165 165 165 F F F 4 080-099 98 98 98 167 167 167 F F F 5 100-119 96 96 96 165 165 165 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 98 98 98 167 167 167 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 94 94 94 160 160 160 F F F 16 320-339 98 98 98 170 170 170 F F F 17 340-359 98 98 98 170 170 170 F F F 18 360-379 98 98 98 168 168 168 F F F 19 380-399 96 96 96 164 164 164 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 98 98 98 167 167 167 T T T 22 440-459 92 92 92 158 158 158 F F F 23 460-479 98 98 98 167 167 167 F F F 24 480-499 98 98 98 169 169 169 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 98 98 98 169 169 169 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 95 95 95 161 161 161 F F F Max gen: 98, Max flush: 170 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 98 98 98 167 167 167 F F F 2 040-059 98 98 98 167 167 167 F F F 3 060-079 96 96 96 165 165 165 F F F 4 080-099 98 98 98 167 167 167 F F F 5 100-119 96 96 96 165 165 165 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 98 98 98 167 167 167 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 92 92 92 156 156 156 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 94 94 94 160 160 160 F F F 16 320-339 98 98 98 170 170 170 F F F 17 340-359 98 98 98 170 170 170 F F F 18 360-379 98 98 98 168 168 168 F F F 19 380-399 96 96 96 164 164 164 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 99 99 99 171 171 171 F F F 22 440-459 92 92 92 158 158 158 F F F 23 460-479 98 98 98 167 167 167 F F F 24 480-499 98 98 98 169 169 169 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 98 98 98 169 169 169 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 95 95 95 161 161 161 F F F Max gen: 99, Max flush: 171 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.250995226Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.294669256Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.295436523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.295477576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.295491993Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.29550652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582} {"msg":"Crucible 22b6f5a2-34ba-42d6-a44f-fb3c39b7e266 has session id: dd79831b-1e96-44c1-aed7-e24a7372311f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.29596676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Upstairs opts: Upstairs UUID: 22b6f5a2-34ba-42d6-a44f-fb3c39b7e266, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.296012532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Crucible stats registered with UUID: 22b6f5a2-34ba-42d6-a44f-fb3c39b7e266","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.296039068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.32659736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582} {"msg":"22b6f5a2-34ba-42d6-a44f-fb3c39b7e266 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326639543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.32665704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326684375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326701952Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326719009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326735676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326752743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.32676889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326786037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326803164Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47616)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326902727Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48985)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.326966546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_connection connected from Ok(127.0.0.1:32972)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.327026725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.327888715Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.327929438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.327967172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.328001616Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.328028781Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.328057826Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.328543861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.328584724Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.328673429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329120501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329148616Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329163783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329176741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329190349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329203487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329270335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329289861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[0]R flush_numbers[0..12]: [161, 167, 167, 165, 167, 165, 152, 161, 164, 166, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329308728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[0]R generation[0..12]: [95, 98, 98, 96, 98, 96, 89, 95, 96, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329323096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329337113Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[1]R flush_numbers[0..12]: [161, 167, 167, 165, 167, 165, 152, 161, 164, 166, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329350911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[1]R generation[0..12]: [95, 98, 98, 96, 98, 96, 89, 95, 96, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329364448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329378336Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[2]R flush_numbers[0..12]: [161, 167, 167, 165, 167, 165, 152, 161, 164, 166, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329391854Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[2]R generation[0..12]: [95, 98, 98, 96, 98, 96, 89, 95, 96, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329405301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329418929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329431787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329444685Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329457652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.32947044Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"":"downstairs","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329483528Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:13:50.329495496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"0","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329521431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"1","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329534589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"client":"2","":"downstairs","session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329547787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329559985Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"22b6f5a2-34ba-42d6-a44f-fb3c39b7e266 is now active with session: dd79831b-1e96-44c1-aed7-e24a7372311f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329572782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.32958458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582,"session_id":"dd79831b-1e96-44c1-aed7-e24a7372311f"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:50.329596848Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 537 len 28672 data: 8 8 10 12 10 9 8 02/30 Read block 337 len 24576 03/30 Write block 395 len 40960 data: 5 5 6 6 6 7 7 10 9 11 04/30 Flush 05/30 Flush 06/30 Write block 369 len 32768 data: 10 10 11 12 11 12 13 9 07/30 Read block 180 len 4096 08/30 Read block 585 len 8192 09/30 Write block 480 len 4096 data: 9 10/30 Write block 261 len 32768 data: 8 7 7 6 6 7 6 5 11/30 Read block 575 len 8192 12/30 Write block 546 len 8192 data: 10 10 13/30 Write block 592 len 12288 data: 11 11 10 14/30 Flush 15/30 Write block 438 len 36864 data: 5 6 3 3 3 3 4 6 9 16/30 Flush 17/30 Write block 317 len 12288 data: 5 6 5 18/30 Read block 90 len 4096 19/30 Write block 467 len 36864 data: 7 8 7 5 7 8 5 7 10 20/30 Flush 21/30 Write block 124 len 4096 data: 12 22/30 Write block 55 len 16384 data: 8 7 9 9 23/30 Write block 35 len 32768 data: 5 5 6 6 7 6 8 7 24/30 Flush 25/30 Write block 74 len 20480 data: 12 12 12 7 7 26/30 Write block 62 len 4096 data: 8 27/30 Write block 310 len 24576 data: 11 10 9 6 6 6 28/30 Read block 163 len 8192 29/30 Read block 410 len 32768 30/30 Read block 462 len 16384 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:50.357852831Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 100 100 100 177 177 177 F F F 2 040-059 100 100 100 177 177 177 F F F 3 060-079 96 96 96 165 165 165 T T T 4 080-099 98 98 98 167 167 167 F F F 5 100-119 96 96 96 165 165 165 F F F 6 120-139 100 100 100 177 177 177 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 98 98 98 167 167 167 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 100 100 100 176 176 176 T T T 16 320-339 98 98 98 170 170 170 F F F 17 340-359 98 98 98 170 170 170 F F F 18 360-379 100 100 100 174 174 174 F F F 19 380-399 100 100 100 172 172 172 F F F 20 400-419 100 100 100 172 172 172 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 100 100 100 175 175 175 F F F 23 460-479 100 100 100 176 176 176 F F F 24 480-499 100 100 100 174 174 174 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 100 100 100 172 172 172 F F F 27 540-559 100 100 100 174 174 174 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 100 100 100 174 174 174 F F F Max gen: 100, Max flush: 177 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 100 100 100 177 177 177 F F F 2 040-059 100 100 100 177 177 177 F F F 3 060-079 101 101 101 178 178 178 F F F 4 080-099 98 98 98 167 167 167 F F F 5 100-119 96 96 96 165 165 165 F F F 6 120-139 100 100 100 177 177 177 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 98 98 98 167 167 167 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 101 101 101 178 178 178 F F F 16 320-339 98 98 98 170 170 170 F F F 17 340-359 98 98 98 170 170 170 F F F 18 360-379 100 100 100 174 174 174 F F F 19 380-399 100 100 100 172 172 172 F F F 20 400-419 100 100 100 172 172 172 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 100 100 100 175 175 175 F F F 23 460-479 100 100 100 176 176 176 F F F 24 480-499 100 100 100 174 174 174 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 100 100 100 172 172 172 F F F 27 540-559 100 100 100 174 174 174 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 100 100 100 174 174 174 F F F Max gen: 101, Max flush: 178 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.643698967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.688542074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.68942532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.689478791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.689499177Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.689514194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590} {"msg":"Crucible 6245e33c-d6e2-4f0e-8e5c-d344e2dcdcaf has session id: 1bec082a-4a9f-4a8a-bdc9-700b4255247e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.68999809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Upstairs opts: Upstairs UUID: 6245e33c-d6e2-4f0e-8e5c-d344e2dcdcaf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.690048571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Crucible stats registered with UUID: 6245e33c-d6e2-4f0e-8e5c-d344e2dcdcaf","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.690076496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721593732Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590} {"msg":"6245e33c-d6e2-4f0e-8e5c-d344e2dcdcaf active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72166042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721692795Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721737347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72177564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721818983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721853657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72188982Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721924994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721959208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.721990113Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48643)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.722020968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57815)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.722052902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46384)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.722081697Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.722895735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72292705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.722941527Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.722955095Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.723171447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.723207281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.723756126Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.723798088Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.723831182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724493087Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724519743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72453455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724618365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724637932Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72465169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724663858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724677385Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[0]R flush_numbers[0..12]: [161, 177, 177, 178, 167, 165, 177, 161, 164, 166, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724737955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[0]R generation[0..12]: [95, 100, 100, 101, 98, 96, 100, 95, 96, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724757701Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724772109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[1]R flush_numbers[0..12]: [161, 177, 177, 178, 167, 165, 177, 161, 164, 166, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724786156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[1]R generation[0..12]: [95, 100, 100, 101, 98, 96, 100, 95, 96, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724803103Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724817121Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[2]R flush_numbers[0..12]: [161, 177, 177, 178, 167, 165, 177, 161, 164, 166, 167, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724830819Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[2]R generation[0..12]: [95, 100, 100, 101, 98, 96, 100, 95, 96, 97, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724844346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724857994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724871112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724884699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724897647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724910445Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"":"downstairs","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724923762Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72493598Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"0","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-09-27T17:13:54.724961136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"1","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724974394Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"client":"2","":"downstairs","session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.724987471Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.725010287Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"6245e33c-d6e2-4f0e-8e5c-d344e2dcdcaf is now active with session: 1bec082a-4a9f-4a8a-bdc9-700b4255247e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.725030484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.725043311Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590,"session_id":"1bec082a-4a9f-4a8a-bdc9-700b4255247e"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:54.72505542Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 445 len 32768 data: 7 10 10 9 10 12 11 9 02/30 Read block 509 len 12288 03/30 Write block 316 len 12288 data: 6 6 7 04/30 Read block 188 len 24576 05/30 Flush 06/30 Write block 398 len 24576 data: 7 7 8 8 11 10 07/30 Flush 08/30 Write block 99 len 8192 data: 11 10 09/30 Write block 454 len 16384 data: 12 10 12 10 10/30 Read block 106 len 32768 11/30 Read block 165 len 8192 12/30 Flush 13/30 Write block 581 len 4096 data: 11 14/30 Flush 15/30 Flush 16/30 Write block 526 len 4096 data: 6 17/30 Read block 329 len 32768 18/30 Read block 344 len 12288 19/30 Read block 367 len 4096 20/30 Write block 405 len 20480 data: 11 12 12 11 9 21/30 Write block 204 len 28672 data: 9 5 5 6 6 9 8 22/30 Write block 123 len 24576 data: 11 13 10 9 10 7 23/30 Read block 99 len 4096 24/30 Read block 310 len 16384 25/30 Write block 300 len 8192 data: 3 3 26/30 Read block 65 len 16384 27/30 Read block 247 len 28672 28/30 Read block 113 len 16384 29/30 Write block 331 len 16384 data: 8 8 8 9 30/30 Read block 385 len 16384 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 1 Done Done Done false 17 Acked 1016 Read 8 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Read 1 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 4 Done Done Sent false 30 Acked 1029 Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 15 14 13 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:54.749786776Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 100 100 100 177 177 177 F F F 2 040-059 100 100 100 177 177 177 F F F 3 060-079 101 101 101 178 178 178 F F F 4 080-099 102 102 102 181 181 181 F F F 5 100-119 102 102 102 181 181 181 F F F 6 120-139 100 100 100 177 177 177 T T T 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 98 98 98 167 167 167 T T T 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 102 102 102 179 179 179 T T T 16 320-339 98 98 98 170 170 170 T T T 17 340-359 98 98 98 170 170 170 F F F 18 360-379 100 100 100 174 174 174 F F F 19 380-399 102 102 102 180 180 180 F F F 20 400-419 102 102 102 180 180 180 T T T 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 100 100 100 176 176 176 F F F 24 480-499 100 100 100 174 174 174 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 100 100 100 172 172 172 T T T 27 540-559 100 100 100 174 174 174 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 102, Max flush: 182 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 161 161 161 F F F 1 020-039 100 100 100 177 177 177 F F F 2 040-059 100 100 100 177 177 177 F F F 3 060-079 101 101 101 178 178 178 F F F 4 080-099 102 102 102 181 181 181 F F F 5 100-119 102 102 102 181 181 181 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 95 95 95 161 161 161 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 103 103 103 183 183 183 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 103 103 103 183 183 183 F F F 16 320-339 103 103 103 183 183 183 F F F 17 340-359 98 98 98 170 170 170 F F F 18 360-379 100 100 100 174 174 174 F F F 19 380-399 102 102 102 180 180 180 F F F 20 400-419 103 103 103 183 183 183 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 100 100 100 176 176 176 F F F 24 480-499 100 100 100 174 174 174 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 103 103 103 183 183 183 F F F 27 540-559 100 100 100 174 174 174 F F F 28 560-579 92 92 92 158 158 158 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 103, Max flush: 183 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.092135169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.135818597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.136626657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.136673818Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.136688346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.136703133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598} {"msg":"Crucible 5e529981-7dbf-4ea6-8c94-a508dfff7513 has session id: 37d89f12-58c6-4a56-8c35-c0e2603a7145","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.137228232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Upstairs opts: Upstairs UUID: 5e529981-7dbf-4ea6-8c94-a508dfff7513, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.137271624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Crucible stats registered with UUID: 5e529981-7dbf-4ea6-8c94-a508dfff7513","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.137288531Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168219709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598} {"msg":"5e529981-7dbf-4ea6-8c94-a508dfff7513 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168257142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168276049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168304614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168322651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168339938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168356775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168377861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168401577Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168435441Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168465526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_connection connected from Ok(127.0.0.1:52030)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.16849766Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_connection connected from Ok(127.0.0.1:52311)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168626718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_connection connected from Ok(127.0.0.1:58044)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.168780181Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.169223174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.169255499Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.169282444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.169307699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.169561925Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.16958953Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170103241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170146713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170180898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170722143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170760957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170788582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170804789Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170818807Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170831964Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170843922Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.17085716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[0]R flush_numbers[0..12]: [161, 177, 177, 178, 181, 181, 183, 161, 164, 166, 183, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170871067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[0]R generation[0..12]: [95, 100, 100, 101, 102, 102, 103, 95, 96, 97, 103, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170885215Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170899133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[1]R flush_numbers[0..12]: [161, 177, 177, 178, 181, 181, 183, 161, 164, 166, 183, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.17091268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[1]R generation[0..12]: [95, 100, 100, 101, 102, 102, 103, 95, 96, 97, 103, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170926378Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170940305Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[2]R flush_numbers[0..12]: [161, 177, 177, 178, 181, 181, 183, 161, 164, 166, 183, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.170956983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"[2]R generation[0..12]: [95, 100, 100, 101, 102, 102, 103, 95, 96, 97, 103, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.17097077Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171000915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.17102653Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171051626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171075572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171097658Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"":"downstairs","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171119494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.1711425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"0","":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171171915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"1","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.17119771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"client":"2","":"downstairs","session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171215407Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171228195Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"5e529981-7dbf-4ea6-8c94-a508dfff7513 is now active with session: 37d89f12-58c6-4a56-8c35-c0e2603a7145","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171241283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171253161Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598,"session_id":"37d89f12-58c6-4a56-8c35-c0e2603a7145"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:59.171267868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 478 len 40960 data: 7 8 10 8 6 8 7 7 7 6 02/30 Write block 10 len 4096 data: 10 03/30 Flush 04/30 Write block 62 len 20480 data: 9 7 6 7 9 05/30 Read block 171 len 36864 06/30 Write block 201 len 40960 data: 10 10 9 10 6 6 7 7 10 9 07/30 Read block 541 len 12288 08/30 Write block 472 len 16384 data: 9 6 8 11 09/30 Write block 163 len 28672 data: 12 12 9 10 9 10 8 10/30 Read block 4 len 36864 11/30 Write block 567 len 16384 data: 8 7 7 6 12/30 Read block 371 len 24576 13/30 Read block 13 len 12288 14/30 Write block 302 len 12288 data: 4 5 6 15/30 Read block 311 len 32768 16/30 Write block 549 len 8192 data: 12 15 17/30 Write block 541 len 24576 data: 11 10 9 9 8 11 18/30 Write block 376 len 12288 data: 10 7 7 19/30 Read block 400 len 4096 20/30 Flush 21/30 Write block 338 len 24576 data: 14 13 14 15 13 12 22/30 Write block 571 len 32768 data: 8 10 10 10 11 10 9 9 23/30 Read block 561 len 4096 24/30 Read block 565 len 12288 25/30 Read block 268 len 36864 26/30 Read block 341 len 40960 27/30 Read block 310 len 36864 28/30 Read block 588 len 4096 29/30 Write block 155 len 16384 data: 7 6 7 7 30/30 Write block 161 len 20480 data: 7 10 13 13 10 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 4 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:59.196180373Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 100 100 100 177 177 177 F F F 2 040-059 100 100 100 177 177 177 F F F 3 060-079 104 104 104 185 185 185 F F F 4 080-099 102 102 102 181 181 181 F F F 5 100-119 102 102 102 181 181 181 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 95 95 95 161 161 161 T T T 8 160-179 104 104 104 185 185 185 T T T 9 180-199 97 97 97 166 166 166 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 104 104 104 185 185 185 F F F 16 320-339 103 103 103 183 183 183 T T T 17 340-359 98 98 98 170 170 170 T T T 18 360-379 104 104 104 185 185 185 F F F 19 380-399 102 102 102 180 180 180 F F F 20 400-419 103 103 103 183 183 183 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 104 104 104 185 185 185 F F F 24 480-499 104 104 104 184 184 184 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 103 103 103 183 183 183 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 104 104 104 185 185 185 T T T 29 580-599 102 102 102 182 182 182 F F F Max gen: 104, Max flush: 185 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 100 100 100 177 177 177 F F F 2 040-059 100 100 100 177 177 177 F F F 3 060-079 104 104 104 185 185 185 F F F 4 080-099 102 102 102 181 181 181 F F F 5 100-119 102 102 102 181 181 181 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 105 105 105 186 186 186 F F F 8 160-179 105 105 105 186 186 186 F F F 9 180-199 97 97 97 166 166 166 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 92 92 92 156 156 156 F F F 15 300-319 104 104 104 185 185 185 F F F 16 320-339 105 105 105 186 186 186 F F F 17 340-359 105 105 105 186 186 186 F F F 18 360-379 104 104 104 185 185 185 F F F 19 380-399 102 102 102 180 180 180 F F F 20 400-419 103 103 103 183 183 183 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 104 104 104 185 185 185 F F F 24 480-499 104 104 104 184 184 184 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 103 103 103 183 183 183 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 105 105 105 186 186 186 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 105, Max flush: 186 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.515337294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:03.560153265Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.560658717Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.593086404Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.593952713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.593990067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.594015942Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.594034169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606} {"msg":"Crucible a5d4269f-9520-41ab-ac30-f0c7372bdd8c has session id: de6e14a6-04df-4589-af13-8c46a546187e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.594564477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Upstairs opts: Upstairs UUID: a5d4269f-9520-41ab-ac30-f0c7372bdd8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.594620897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Crucible stats registered with UUID: a5d4269f-9520-41ab-ac30-f0c7372bdd8c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.594642673Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625153424Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606} {"msg":"a5d4269f-9520-41ab-ac30-f0c7372bdd8c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625197226Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625214673Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625242158Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625259815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625276962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625293749Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625310576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625335342Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625370916Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625399721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60133)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625538177Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48006)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625586748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34992)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.625618393Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.626385869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.626419963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.62643993Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.626459126Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.626626467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.626671589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627104234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627138398Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627405741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627735364Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627768078Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627783846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627796824Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627810541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.627823779Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628013966Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628035192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[0]R flush_numbers[0..12]: [184, 177, 177, 185, 181, 181, 183, 186, 186, 166, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628049649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[0]R generation[0..12]: [104, 100, 100, 104, 102, 102, 103, 105, 105, 97, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628063757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628078044Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[1]R flush_numbers[0..12]: [184, 177, 177, 185, 181, 181, 183, 186, 186, 166, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628091772Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[1]R generation[0..12]: [104, 100, 100, 104, 102, 102, 103, 105, 105, 97, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.6281054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628119217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[2]R flush_numbers[0..12]: [184, 177, 177, 185, 181, 181, 183, 186, 186, 166, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628132975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[2]R generation[0..12]: [104, 100, 100, 104, 102, 102, 103, 105, 105, 97, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628150002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.62816409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628177277Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628190375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628203153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.62821618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"":"downstairs","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628229248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:14:03.628241386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"0","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628261833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"1","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.62827493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"client":"2","":"downstairs","session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628288048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628300376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"a5d4269f-9520-41ab-ac30-f0c7372bdd8c is now active with session: de6e14a6-04df-4589-af13-8c46a546187e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628313224Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628325441Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606,"session_id":"de6e14a6-04df-4589-af13-8c46a546187e"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:03.628337619Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 105 len 40960 data: 8 11 9 8 7 8 7 8 5 7 02/30 Write block 333 len 40960 data: 9 10 14 13 13 15 14 15 16 14 03/30 Read block 29 len 12288 04/30 Flush 05/30 Flush 06/30 Write block 473 len 24576 data: 7 9 12 8 6 8 07/30 Read block 331 len 20480 08/30 Read block 509 len 24576 09/30 Write block 34 len 32768 data: 8 6 6 7 7 8 7 9 10/30 Read block 231 len 4096 11/30 Read block 120 len 24576 12/30 Write block 161 len 8192 data: 8 11 13/30 Flush 14/30 Write block 270 len 12288 data: 4 5 5 15/30 Read block 93 len 12288 16/30 Read block 96 len 28672 17/30 Read block 464 len 4096 18/30 Read block 2 len 28672 19/30 Write block 533 len 4096 data: 8 20/30 Read block 340 len 40960 21/30 Write block 31 len 20480 data: 7 7 8 9 7 22/30 Write block 358 len 28672 data: 5 9 9 12 15 14 13 23/30 Read block 215 len 20480 24/30 Read block 339 len 36864 25/30 Read block 437 len 4096 26/30 Write block 182 len 20480 data: 7 7 7 8 8 27/30 Read block 323 len 12288 28/30 Write block 290 len 36864 data: 7 7 7 5 5 3 3 4 3 29/30 Read block 383 len 16384 30/30 Write block 38 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 3 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 7 Done Done Done false 17 Acked 1016 Read 1 Done Done Done false 18 Acked 1017 Read 7 Done Done Done false 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:03.654105635Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 106 106 106 189 189 189 T T T 2 040-059 106 106 106 189 189 189 F F F 3 060-079 104 104 104 185 185 185 F F F 4 080-099 102 102 102 181 181 181 F F F 5 100-119 106 106 106 187 187 187 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 105 105 105 186 186 186 F F F 8 160-179 106 106 106 189 189 189 F F F 9 180-199 97 97 97 166 166 166 T T T 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 100 100 100 174 174 174 T T T 14 280-299 92 92 92 156 156 156 T T T 15 300-319 104 104 104 185 185 185 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 106 106 106 187 187 187 T T T 18 360-379 104 104 104 185 185 185 T T T 19 380-399 102 102 102 180 180 180 F F F 20 400-419 103 103 103 183 183 183 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 106 106 106 189 189 189 F F F 24 480-499 104 104 104 184 184 184 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 103 103 103 183 183 183 T T T 27 540-559 104 104 104 185 185 185 F F F 28 560-579 105 105 105 186 186 186 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 106, Max flush: 189 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 107 107 107 190 190 190 F F F 2 040-059 106 106 106 189 189 189 F F F 3 060-079 104 104 104 185 185 185 F F F 4 080-099 102 102 102 181 181 181 F F F 5 100-119 106 106 106 187 187 187 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 105 105 105 186 186 186 F F F 8 160-179 106 106 106 189 189 189 F F F 9 180-199 107 107 107 190 190 190 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 104 104 104 185 185 185 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 107 107 107 190 190 190 F F F 18 360-379 107 107 107 190 190 190 F F F 19 380-399 102 102 102 180 180 180 F F F 20 400-419 103 103 103 183 183 183 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 106 106 106 189 189 189 F F F 24 480-499 104 104 104 184 184 184 F F F 25 500-519 98 98 98 169 169 169 F F F 26 520-539 107 107 107 190 190 190 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 105 105 105 186 186 186 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 107, Max flush: 190 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.039698244Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.084448317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.085409339Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.085447133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.08546138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.085480697Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614} {"msg":"Crucible 2a696e60-b3ea-4d6c-8550-a940a9d4ed1d has session id: f843aa43-7454-4300-a6bf-73de2297e097","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.085990908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Upstairs opts: Upstairs UUID: 2a696e60-b3ea-4d6c-8550-a940a9d4ed1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.08609541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Crucible stats registered with UUID: 2a696e60-b3ea-4d6c-8550-a940a9d4ed1d","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.086125665Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117237681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614} {"msg":"2a696e60-b3ea-4d6c-8550-a940a9d4ed1d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117292091Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117314787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117367928Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117406541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117441125Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117475949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117511533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117546197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.11758772Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117636511Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_connection connected from Ok(127.0.0.1:57744)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117681314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_connection connected from Ok(127.0.0.1:34071)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.117769708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_connection connected from Ok(127.0.0.1:56959)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.11781706Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.118530316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.118557251Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.118571528Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.118585056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.118761115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.118827794Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.119288284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.119316389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.119331476Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.119901887Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.119924153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.11994243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120046422Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120076307Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120091564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120104502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120118699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[0]R flush_numbers[0..12]: [184, 190, 189, 185, 181, 187, 183, 186, 189, 190, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120132727Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[0]R generation[0..12]: [104, 107, 106, 104, 102, 106, 103, 105, 106, 107, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120147194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120161512Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[1]R flush_numbers[0..12]: [184, 190, 189, 185, 181, 187, 183, 186, 189, 190, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120185818Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[1]R generation[0..12]: [104, 107, 106, 104, 102, 106, 103, 105, 106, 107, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120200375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120216412Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[2]R flush_numbers[0..12]: [184, 190, 189, 185, 181, 187, 183, 186, 189, 190, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.12023027Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[2]R generation[0..12]: [104, 107, 106, 104, 102, 106, 103, 105, 106, 107, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120244417Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120258925Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120276662Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120290569Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120303607Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120316525Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"":"downstairs","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120329682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120341631Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"0","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120368246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"1","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120381464Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"client":"2","":"downstairs","session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120394691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120407069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"2a696e60-b3ea-4d6c-8550-a940a9d4ed1d is now active with session: f843aa43-7454-4300-a6bf-73de2297e097","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120419967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120432295Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614,"session_id":"f843aa43-7454-4300-a6bf-73de2297e097"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:08.120444802Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 4096 data: 6 02/30 Read block 104 len 8192 03/30 Read block 326 len 8192 04/30 Read block 527 len 4096 05/30 Read block 211 len 24576 06/30 Flush 07/30 Write block 63 len 28672 data: 8 7 8 10 9 9 10 08/30 Write block 240 len 24576 data: 8 9 8 8 6 6 09/30 Read block 502 len 20480 10/30 Read block 349 len 16384 11/30 Write block 514 len 4096 data: 10 12/30 Write block 397 len 32768 data: 7 8 8 9 9 12 11 12 13/30 Write block 165 len 28672 data: 11 11 10 11 9 7 8 14/30 Read block 350 len 4096 15/30 Read block 97 len 32768 16/30 Read block 486 len 20480 17/30 Write block 310 len 8192 data: 12 11 18/30 Read block 448 len 12288 19/30 Write block 405 len 16384 data: 12 13 13 12 20/30 Write block 366 len 24576 data: 11 11 9 11 11 12 21/30 Read block 28 len 28672 22/30 Read block 108 len 24576 23/30 Write block 516 len 36864 data: 15 14 11 9 9 10 11 10 11 24/30 Write block 411 len 4096 data: 9 25/30 Write block 534 len 24576 data: 8 10 8 9 9 11 26/30 Read block 312 len 24576 27/30 Read block 442 len 36864 28/30 Write block 562 len 16384 data: 8 6 7 8 29/30 Read block 294 len 32768 30/30 Write block 179 len 40960 data: 6 8 6 8 8 8 9 9 9 8 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 7 Acked 1006 Write 7 Done Done Done false 8 Acked 1007 Write 6 Done Done Done false 9 Acked 1008 Read 5 Done Done Done false 10 Acked 1009 Read 4 Done Done Done false 11 Acked 1010 Write 1 Done Done Done false 12 Acked 1011 Write 8 Done Done Done false 13 Acked 1012 Write 7 Done Done Done false 14 Acked 1013 Read 1 Done Done Done false 15 Acked 1014 Read 8 Done Done Done false 16 Acked 1015 Read 5 Done Done Done false 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Write 6 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:08.141705773Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 107 107 107 190 190 190 F F F 2 040-059 106 106 106 189 189 189 F F F 3 060-079 104 104 104 185 185 185 T T T 4 080-099 102 102 102 181 181 181 F F F 5 100-119 106 106 106 187 187 187 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 105 105 105 186 186 186 F F F 8 160-179 106 106 106 189 189 189 T T T 9 180-199 107 107 107 190 190 190 T T T 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 92 92 92 156 156 156 T T T 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 104 104 104 185 185 185 T T T 16 320-339 106 106 106 187 187 187 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 107 107 107 190 190 190 T T T 19 380-399 102 102 102 180 180 180 T T T 20 400-419 103 103 103 183 183 183 T T T 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 106 106 106 189 189 189 F F F 24 480-499 104 104 104 184 184 184 F F F 25 500-519 98 98 98 169 169 169 T T T 26 520-539 107 107 107 190 190 190 T T T 27 540-559 104 104 104 185 185 185 F F F 28 560-579 105 105 105 186 186 186 T T T 29 580-599 102 102 102 182 182 182 F F F Max gen: 108, Max flush: 191 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 107 107 107 190 190 190 F F F 2 040-059 106 106 106 189 189 189 F F F 3 060-079 109 109 109 192 192 192 F F F 4 080-099 102 102 102 181 181 181 F F F 5 100-119 106 106 106 187 187 187 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 105 105 105 186 186 186 F F F 8 160-179 109 109 109 192 192 192 F F F 9 180-199 109 109 109 192 192 192 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 109 109 109 192 192 192 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 109 109 109 192 192 192 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 109 109 109 192 192 192 F F F 19 380-399 109 109 109 192 192 192 F F F 20 400-419 109 109 109 192 192 192 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 102 102 102 181 181 181 F F F 23 460-479 106 106 106 189 189 189 F F F 24 480-499 104 104 104 184 184 184 F F F 25 500-519 109 109 109 192 192 192 F F F 26 520-539 109 109 109 192 192 192 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 109, Max flush: 192 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.55408679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.598545324Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.599546489Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.599585533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.599605319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.599620236Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622} {"msg":"Crucible 4abd3ce1-4423-44d7-8cc4-e7135c663877 has session id: 23e8b9a2-3e21-4be3-bdc8-8b94ac299908","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.60022981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Upstairs opts: Upstairs UUID: 4abd3ce1-4423-44d7-8cc4-e7135c663877, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.600276572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Crucible stats registered with UUID: 4abd3ce1-4423-44d7-8cc4-e7135c663877","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.600295529Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631323939Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622} {"msg":"4abd3ce1-4423-44d7-8cc4-e7135c663877 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.63137845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631398546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631429461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631447808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631470464Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631501079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631534133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631567517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631599551Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631637425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_connection connected from Ok(127.0.0.1:64606)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631673159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_connection connected from Ok(127.0.0.1:52988)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631709702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_connection connected from Ok(127.0.0.1:46092)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.631743977Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.632516342Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.632546877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.632561624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.632580791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.632783605Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.632812331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633258873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633289837Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633418475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633873996Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633914479Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633940184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633954072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633967699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633980837Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.633993025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634007352Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[0]R flush_numbers[0..12]: [184, 190, 189, 192, 181, 187, 183, 186, 192, 192, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634051145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[0]R generation[0..12]: [104, 107, 106, 109, 102, 106, 103, 105, 109, 109, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634069812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634084539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[1]R flush_numbers[0..12]: [184, 190, 189, 192, 181, 187, 183, 186, 192, 192, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634098407Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[1]R generation[0..12]: [104, 107, 106, 109, 102, 106, 103, 105, 109, 109, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634111914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634125772Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[2]R flush_numbers[0..12]: [184, 190, 189, 192, 181, 187, 183, 186, 192, 192, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.63413931Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[2]R generation[0..12]: [104, 107, 106, 109, 102, 106, 103, 105, 109, 109, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634152987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634169064Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634182132Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.63419552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634208688Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634221835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"":"downstairs","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634235053Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:14:12.634247001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"0","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634267557Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"1","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634280595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"client":"2","":"downstairs","session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634293603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634305751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"4abd3ce1-4423-44d7-8cc4-e7135c663877 is now active with session: 23e8b9a2-3e21-4be3-bdc8-8b94ac299908","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634318478Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634330306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622,"session_id":"23e8b9a2-3e21-4be3-bdc8-8b94ac299908"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:12.634342444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 40960 data: 7 9 11 12 10 8 11 11 13 14 02/30 Write block 514 len 12288 data: 11 13 16 03/30 Read block 415 len 24576 04/30 Read block 552 len 16384 05/30 Write block 153 len 24576 data: 5 6 8 7 8 8 06/30 Read block 68 len 24576 07/30 Write block 484 len 32768 data: 8 8 8 7 5 6 5 6 08/30 Write block 457 len 28672 data: 11 10 7 7 8 9 11 09/30 Write block 502 len 8192 data: 8 7 10/30 Flush 11/30 Write block 0 len 32768 data: 2 2 4 4 8 10 12 13 12/30 Write block 151 len 16384 data: 7 7 6 7 13/30 Write block 396 len 24576 data: 6 8 9 9 10 10 14/30 Read block 508 len 4096 15/30 Write block 60 len 4096 data: 8 16/30 Write block 460 len 20480 data: 8 9 10 12 12 17/30 Read block 178 len 36864 18/30 Read block 312 len 12288 19/30 Write block 494 len 8192 data: 5 5 20/30 Read block 336 len 36864 21/30 Read block 483 len 8192 22/30 Read block 343 len 32768 23/30 Write block 22 len 20480 data: 5 5 6 6 6 24/30 Flush 25/30 Write block 97 len 36864 data: 10 11 12 11 10 11 12 9 9 26/30 Write block 518 len 20480 data: 12 10 10 11 12 27/30 Write block 5 len 8192 data: 11 13 28/30 Write block 518 len 4096 data: 13 29/30 Write block 442 len 32768 data: 4 4 5 8 11 11 10 11 30/30 Read block 85 len 20480 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 8 Sent Done Sent false 30 Acked 1029 Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 4 6 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:12.658842001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 194 194 194 T T T 1 020-039 110 110 110 194 194 194 F F F 2 040-059 106 106 106 189 189 189 F F F 3 060-079 110 110 110 194 194 194 F F F 4 080-099 102 102 102 181 181 181 T T T 5 100-119 106 106 106 187 187 187 T T T 6 120-139 103 103 103 183 183 183 F F F 7 140-159 110 110 110 194 194 194 F F F 8 160-179 109 109 109 192 192 192 F F F 9 180-199 109 109 109 192 192 192 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 109 109 109 192 192 192 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 109 109 109 192 192 192 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 109 109 109 192 192 192 F F F 19 380-399 110 110 110 194 194 194 F F F 20 400-419 110 110 110 194 194 194 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 110 110 110 193 193 193 T T T 23 460-479 110 110 110 194 194 194 F F F 24 480-499 110 110 110 194 194 194 F F F 25 500-519 110 110 110 193 193 193 T T T 26 520-539 109 109 109 192 192 192 T T T 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 110, Max flush: 194 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 F F F 1 020-039 110 110 110 194 194 194 F F F 2 040-059 106 106 106 189 189 189 F F F 3 060-079 110 110 110 194 194 194 F F F 4 080-099 111 111 111 195 195 195 F F F 5 100-119 111 111 111 195 195 195 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 110 110 110 194 194 194 F F F 8 160-179 109 109 109 192 192 192 F F F 9 180-199 109 109 109 192 192 192 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 96 96 96 163 163 163 F F F 12 240-259 109 109 109 192 192 192 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 109 109 109 192 192 192 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 109 109 109 192 192 192 F F F 19 380-399 110 110 110 194 194 194 F F F 20 400-419 110 110 110 194 194 194 F F F 21 420-439 100 100 100 175 175 175 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 110 110 110 194 194 194 F F F 24 480-499 110 110 110 194 194 194 F F F 25 500-519 111 111 111 195 195 195 F F F 26 520-539 111 111 111 195 195 195 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 102 102 102 182 182 182 F F F Max gen: 111, Max flush: 195 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:16.992879222Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:17.035799133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.036302105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.06828137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.069190502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.069235524Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.069253211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.069272208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630} {"msg":"Crucible c4bcab2c-a0b8-4856-aa26-15ec50fd31f1 has session id: 14a2d686-5548-41a8-830c-7a86e9d9b043","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.069755524Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Upstairs opts: Upstairs UUID: c4bcab2c-a0b8-4856-aa26-15ec50fd31f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.069802426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Crucible stats registered with UUID: c4bcab2c-a0b8-4856-aa26-15ec50fd31f1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.069828031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.100855502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630} {"msg":"c4bcab2c-a0b8-4856-aa26-15ec50fd31f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.100913421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.100933218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.100963343Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10098141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.100999897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.101016654Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10103494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.101051448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.101068674Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10109487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_connection connected from Ok(127.0.0.1:45744)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.101299984Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_connection connected from Ok(127.0.0.1:36406)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.101349556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_connection connected from Ok(127.0.0.1:49079)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10138028Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102218625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102255878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102283123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102308529Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102336034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102365249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102946688Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.102979422Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103007947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10356494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103597885Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103614211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103627159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103644686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103658184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103670372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103683659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[0]R flush_numbers[0..12]: [195, 194, 189, 194, 195, 195, 183, 194, 192, 192, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103697637Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[0]R generation[0..12]: [111, 110, 106, 110, 111, 111, 103, 110, 109, 109, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103711905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103726052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[1]R flush_numbers[0..12]: [195, 194, 189, 194, 195, 195, 183, 194, 192, 192, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10373997Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[1]R generation[0..12]: [111, 110, 106, 110, 111, 111, 103, 110, 109, 109, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103753957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103768225Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[2]R flush_numbers[0..12]: [195, 194, 189, 194, 195, 195, 183, 194, 192, 192, 185, 163]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103781882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[2]R generation[0..12]: [111, 110, 106, 110, 111, 111, 103, 110, 109, 109, 104, 96]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10379544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103809268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103822375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103837433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.10385025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-09-27T17:14:17.103865978Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"":"downstairs","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103956702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103975889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"0","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.103991396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"1","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.104015092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"client":"2","":"downstairs","session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.104039417Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.104063583Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"c4bcab2c-a0b8-4856-aa26-15ec50fd31f1 is now active with session: 14a2d686-5548-41a8-830c-7a86e9d9b043","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.104089259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.104104396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630,"session_id":"14a2d686-5548-41a8-830c-7a86e9d9b043"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:17.104116694Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 8192 data: 11 11 02/30 Read block 6 len 28672 03/30 Write block 56 len 24576 data: 8 10 10 7 9 8 04/30 Write block 509 len 4096 data: 10 05/30 Write block 497 len 20480 data: 8 8 8 6 7 06/30 Read block 523 len 40960 07/30 Write block 530 len 16384 data: 11 8 8 9 08/30 Read block 56 len 20480 09/30 Flush 10/30 Write block 66 len 4096 data: 11 11/30 Read block 369 len 4096 12/30 Read block 583 len 28672 13/30 Write block 233 len 20480 data: 7 8 10 9 9 14/30 Write block 595 len 8192 data: 8 7 15/30 Read block 171 len 24576 16/30 Read block 450 len 28672 17/30 Read block 584 len 8192 18/30 Flush 19/30 Write block 414 len 24576 data: 9 11 9 9 6 7 20/30 Read block 270 len 20480 21/30 Read block 377 len 4096 22/30 Write block 34 len 40960 data: 10 8 7 8 9 9 8 10 8 8 23/30 Read block 462 len 16384 24/30 Read block 179 len 40960 25/30 Read block 128 len 16384 26/30 Read block 191 len 4096 27/30 Read block 247 len 8192 28/30 Write block 429 len 24576 data: 7 9 9 10 9 10 29/30 Read block 484 len 32768 30/30 Read block 236 len 4096 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:17.128117068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 F F F 1 020-039 110 110 110 194 194 194 T T T 2 040-059 112 112 112 196 196 196 T T T 3 060-079 112 112 112 197 197 197 F F F 4 080-099 111 111 111 195 195 195 F F F 5 100-119 111 111 111 195 195 195 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 110 110 110 194 194 194 F F F 8 160-179 109 109 109 192 192 192 F F F 9 180-199 109 109 109 192 192 192 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 112 112 112 197 197 197 F F F 12 240-259 109 109 109 192 192 192 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 109 109 109 192 192 192 F F F 19 380-399 110 110 110 194 194 194 F F F 20 400-419 110 110 110 194 194 194 T T T 21 420-439 100 100 100 175 175 175 T T T 22 440-459 111 111 111 195 195 195 F F F 23 460-479 110 110 110 194 194 194 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 112 112 112 196 196 196 F F F 26 520-539 112 112 112 196 196 196 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 112, Max flush: 197 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 F F F 1 020-039 113 113 113 198 198 198 F F F 2 040-059 113 113 113 198 198 198 F F F 3 060-079 112 112 112 197 197 197 F F F 4 080-099 111 111 111 195 195 195 F F F 5 100-119 111 111 111 195 195 195 F F F 6 120-139 103 103 103 183 183 183 F F F 7 140-159 110 110 110 194 194 194 F F F 8 160-179 109 109 109 192 192 192 F F F 9 180-199 109 109 109 192 192 192 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 112 112 112 197 197 197 F F F 12 240-259 109 109 109 192 192 192 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 109 109 109 192 192 192 F F F 19 380-399 110 110 110 194 194 194 F F F 20 400-419 113 113 113 198 198 198 F F F 21 420-439 113 113 113 198 198 198 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 110 110 110 194 194 194 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 112 112 112 196 196 196 F F F 26 520-539 112 112 112 196 196 196 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 113, Max flush: 198 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.477018311Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.522341874Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.523321573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.523358987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.523374314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.523388742Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638} {"msg":"Crucible 8823c66f-9181-4040-9de6-5296b37e2b73 has session id: 9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.523846382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Upstairs opts: Upstairs UUID: 8823c66f-9181-4040-9de6-5296b37e2b73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.523891374Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Crucible stats registered with UUID: 8823c66f-9181-4040-9de6-5296b37e2b73","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.523923179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555039124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638} {"msg":"8823c66f-9181-4040-9de6-5296b37e2b73 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555093375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555110872Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555142376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555160533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.55517755Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555194457Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555319025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.55535004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555375715Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555400281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42496)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555425327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:65431)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555451552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:64090)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.555476008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.556314032Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.556348916Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.556486052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.556513628Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.556699735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.556741018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.556955451Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557125971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557410651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557544628Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557565714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557580652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557663877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557683344Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557697002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557944648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.557967695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[0]R flush_numbers[0..12]: [195, 198, 198, 197, 195, 195, 183, 194, 192, 192, 185, 197]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.55799343Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[0]R generation[0..12]: [111, 113, 113, 112, 111, 111, 103, 110, 109, 109, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558008437Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558022845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[1]R flush_numbers[0..12]: [195, 198, 198, 197, 195, 195, 183, 194, 192, 192, 185, 197]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558036812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[1]R generation[0..12]: [111, 113, 113, 112, 111, 111, 103, 110, 109, 109, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.55805034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558064218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[2]R flush_numbers[0..12]: [195, 198, 198, 197, 195, 195, 183, 194, 192, 192, 185, 197]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558078525Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[2]R generation[0..12]: [111, 113, 113, 112, 111, 111, 103, 110, 109, 109, 104, 112]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558092213Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.55810603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558118958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558132006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558144854Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558157481Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"":"downstairs","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558173009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msgWait for a query_work_queue command to finish before sending IO ":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558195175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"0","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558208382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"1","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.5582215Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"client":"2","":"downstairs","session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558234688Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558247026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"8823c66f-9181-4040-9de6-5296b37e2b73 is now active with session: 9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558259793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558271782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638,"session_id":"9ab7eaf9-f80b-42eb-bdd6-e861b6a873a1"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:21.558283809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 455 len 20480 data: 11 13 12 11 8 02/30 Write block 239 len 32768 data: 9 9 10 9 9 7 7 5 03/30 Write block 536 len 12288 data: 9 10 10 04/30 Flush 05/30 Write block 359 len 40960 data: 10 10 13 16 15 14 12 12 12 10 06/30 Flush 07/30 Write block 251 len 12288 data: 7 5 6 08/30 Read block 471 len 24576 09/30 Write block 166 len 12288 data: 12 11 12 10/30 Write block 237 len 40960 data: 10 9 10 10 11 10 10 8 8 6 11/30 Write block 132 len 36864 data: 8 6 8 7 9 9 11 10 8 12/30 Write block 329 len 40960 data: 7 6 9 9 10 11 15 14 14 16 13/30 Read block 119 len 8192 14/30 Read block 348 len 20480 15/30 Write block 224 len 24576 data: 7 7 7 7 7 8 16/30 Read block 220 len 8192 17/30 Read block 272 len 20480 18/30 Read block 212 len 4096 19/30 Read block 379 len 4096 20/30 Read block 294 len 40960 21/30 Flush 22/30 Flush 23/30 Write block 420 len 12288 data: 6 7 8 24/30 Read block 432 len 36864 25/30 Read block 467 len 28672 26/30 Write block 442 len 24576 data: 5 5 6 9 12 12 27/30 Read block 329 len 20480 28/30 Write block 119 len 4096 data: 12 29/30 Read block 148 len 16384 30/30 Write block 364 len 36864 data: 15 13 13 13 11 12 12 13 13 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:21.585105022Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 F F F 1 020-039 113 113 113 198 198 198 F F F 2 040-059 113 113 113 198 198 198 F F F 3 060-079 112 112 112 197 197 197 F F F 4 080-099 111 111 111 195 195 195 F F F 5 100-119 111 111 111 195 195 195 T T T 6 120-139 114 114 114 201 201 201 F F F 7 140-159 114 114 114 201 201 201 F F F 8 160-179 114 114 114 201 201 201 F F F 9 180-199 109 109 109 192 192 192 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 114 114 114 201 201 201 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 114 114 114 201 201 201 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 114 114 114 200 200 200 T T T 19 380-399 110 110 110 194 194 194 F F F 20 400-419 113 113 113 198 198 198 F F F 21 420-439 113 113 113 198 198 198 T T T 22 440-459 114 114 114 199 199 199 T T T 23 460-479 110 110 110 194 194 194 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 112 112 112 196 196 196 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 114, Max flush: 201 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 F F F 1 020-039 113 113 113 198 198 198 F F F 2 040-059 113 113 113 198 198 198 F F F 3 060-079 112 112 112 197 197 197 F F F 4 080-099 111 111 111 195 195 195 F F F 5 100-119 115 115 115 202 202 202 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 114 114 114 201 201 201 F F F 8 160-179 114 114 114 201 201 201 F F F 9 180-199 109 109 109 192 192 192 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 114 114 114 201 201 201 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 114 114 114 201 201 201 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 115 115 115 202 202 202 F F F 19 380-399 110 110 110 194 194 194 F F F 20 400-419 113 113 113 198 198 198 F F F 21 420-439 115 115 115 202 202 202 F F F 22 440-459 115 115 115 202 202 202 F F F 23 460-479 110 110 110 194 194 194 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 112 112 112 196 196 196 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 115, Max flush: 202 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.887557569Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.932164146Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.933055531Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.933093075Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.933108382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.933123099Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646} {"msg":"Crucible 99e4d238-81f2-4fa9-94f8-e04617f2034e has session id: 0f98a156-f7bb-4a0b-b376-b217359abade","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.933745411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Upstairs opts: Upstairs UUID: 99e4d238-81f2-4fa9-94f8-e04617f2034e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.933802841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Crucible stats registered with UUID: 99e4d238-81f2-4fa9-94f8-e04617f2034e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.933829336Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965142717Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646} {"msg":"99e4d238-81f2-4fa9-94f8-e04617f2034e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965193958Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965211455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965248389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965269775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.96529981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965330485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965362459Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965394734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965428498Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965452293Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_connection connected from Ok(127.0.0.1:58829)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.9654702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_connection connected from Ok(127.0.0.1:38708)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965499655Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_connection connected from Ok(127.0.0.1:64461)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.965771558Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.966384631Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.966420105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.966446691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.966475635Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.966676431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.966704186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.96725159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.967294803Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.967325507Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.967897838Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.967928023Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.967946559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968007119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968036864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968052091Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968065699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968079286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[0]R flush_numbers[0..12]: [195, 198, 198, 197, 195, 202, 201, 201, 201, 192, 185, 201]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968136126Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[0]R generation[0..12]: [111, 113, 113, 112, 111, 115, 114, 114, 114, 109, 104, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968155423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.96817336Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[1]R flush_numbers[0..12]: [195, 198, 198, 197, 195, 202, 201, 201, 201, 192, 185, 201]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968187517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[1]R generation[0..12]: [111, 113, 113, 112, 111, 115, 114, 114, 114, 109, 104, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968201405Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968215593Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[2]R flush_numbers[0..12]: [195, 198, 198, 197, 195, 202, 201, 201, 201, 192, 185, 201]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.96822936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[2]R generation[0..12]: [111, 113, 113, 112, 111, 115, 114, 114, 114, 109, 104, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968243068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968256825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968270403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968283771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968296558Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968309356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"":"downstairs","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-09-27T17:14:25.968322784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.96834596Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"0","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968359467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"1","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968372415Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"client":"2","":"downstairs","session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968387603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.96840012Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"99e4d238-81f2-4fa9-94f8-e04617f2034e is now active with session: 0f98a156-f7bb-4a0b-b376-b217359abade","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968413028Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968424976Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646,"session_id":"0f98a156-f7bb-4a0b-b376-b217359abade"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:25.968437184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 28672 data: 8 9 8 7 6 6 8 02/30 Write block 158 len 8192 data: 9 9 03/30 Read block 161 len 24576 04/30 Write block 91 len 8192 data: 4 4 05/30 Read block 122 len 12288 06/30 Read block 333 len 36864 07/30 Read block 166 len 40960 08/30 Read block 52 len 20480 09/30 Read block 454 len 36864 10/30 Read block 240 len 24576 11/30 Read block 533 len 20480 12/30 Read block 508 len 36864 13/30 Read block 197 len 20480 14/30 Read block 220 len 36864 15/30 Read block 436 len 20480 16/30 Read block 262 len 16384 17/30 Flush 18/30 Flush 19/30 Write block 424 len 8192 data: 7 9 20/30 Write block 67 len 16384 data: 10 10 11 10 21/30 Read block 249 len 32768 22/30 Write block 29 len 12288 data: 7 7 8 23/30 Flush 24/30 Write block 370 len 32768 data: 13 14 14 12 13 14 11 8 25/30 Write block 504 len 16384 data: 8 7 7 9 26/30 Read block 121 len 24576 27/30 Write block 168 len 16384 data: 13 10 8 9 28/30 Write block 190 len 28672 data: 10 8 8 5 6 8 10 29/30 Write block 232 len 40960 data: 6 8 9 11 10 11 10 11 11 12 30/30 Write block 62 len 20480 data: 10 9 8 9 12 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 10 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:25.993797461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 F F F 1 020-039 116 116 116 205 205 205 F F F 2 040-059 113 113 113 198 198 198 F F F 3 060-079 116 116 116 205 205 205 T T T 4 080-099 116 116 116 203 203 203 F F F 5 100-119 115 115 115 202 202 202 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 114 114 114 201 201 201 T T T 9 180-199 109 109 109 192 192 192 T T T 10 200-219 104 104 104 185 185 185 F F F 11 220-239 114 114 114 201 201 201 T T T 12 240-259 114 114 114 201 201 201 T T T 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 114 114 114 201 201 201 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 115 115 115 202 202 202 T T T 19 380-399 110 110 110 194 194 194 F F F 20 400-419 113 113 113 198 198 198 F F F 21 420-439 116 116 116 205 205 205 F F F 22 440-459 115 115 115 202 202 202 F F F 23 460-479 110 110 110 194 194 194 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 116 116 116 203 203 203 T T T 26 520-539 114 114 114 199 199 199 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 116, Max flush: 205 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 F F F 1 020-039 116 116 116 205 205 205 F F F 2 040-059 113 113 113 198 198 198 F F F 3 060-079 117 117 117 206 206 206 F F F 4 080-099 116 116 116 203 203 203 F F F 5 100-119 115 115 115 202 202 202 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 117 117 117 206 206 206 F F F 9 180-199 117 117 117 206 206 206 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 117 117 117 206 206 206 F F F 12 240-259 117 117 117 206 206 206 F F F 13 260-279 107 107 107 190 190 190 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 114 114 114 201 201 201 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 117 117 117 206 206 206 F F F 19 380-399 110 110 110 194 194 194 F F F 20 400-419 113 113 113 198 198 198 F F F 21 420-439 116 116 116 205 205 205 F F F 22 440-459 115 115 115 202 202 202 F F F 23 460-479 110 110 110 194 194 194 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 117 117 117 206 206 206 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 117, Max flush: 206 Begin loop 59 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.350507516Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.394586675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.395357961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.395397444Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.395411512Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.395425929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654} {"msg":"Crucible 6dfb4c78-1f8d-4115-8656-c68fc9dbce82 has session id: a0a067cf-c8a4-4d19-a3f3-7b4c0abea324","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.39594075Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Upstairs opts: Upstairs UUID: 6dfb4c78-1f8d-4115-8656-c68fc9dbce82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.395988951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Crucible stats registered with UUID: 6dfb4c78-1f8d-4115-8656-c68fc9dbce82","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.396017326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.426960652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654} {"msg":"6dfb4c78-1f8d-4115-8656-c68fc9dbce82 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427012433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427031589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427063574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427080961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427109906Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.42714156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427177584Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427205599Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427236664Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427270098Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_connection connected from Ok(127.0.0.1:40417)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427319139Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_connection connected from Ok(127.0.0.1:36857)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427358683Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_connection connected from Ok(127.0.0.1:58733)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.427392617Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.428304988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.428345501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.428377425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.428450882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.428482527Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.428511652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429045399Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429081543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429236186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429661752Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429702515Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.42972891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429750446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429777491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429799428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429861647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429881093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[0]R flush_numbers[0..12]: [195, 205, 198, 206, 203, 202, 201, 203, 206, 206, 185, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429905699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[0]R generation[0..12]: [111, 116, 113, 117, 116, 115, 114, 116, 117, 117, 104, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429920917Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429935014Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[1]R flush_numbers[0..12]: [195, 205, 198, 206, 203, 202, 201, 203, 206, 206, 185, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429948902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[1]R generation[0..12]: [111, 116, 113, 117, 116, 115, 114, 116, 117, 117, 104, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429962729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429976807Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[2]R flush_numbers[0..12]: [195, 205, 198, 206, 203, 202, 201, 203, 206, 206, 185, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.429993704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[2]R generation[0..12]: [111, 116, 113, 117, 116, 115, 114, 116, 117, 117, 104, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430008891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430022849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430035707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430048634Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430061452Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.43007429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"":"downstairs","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430087518Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430099695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"0","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2024-09-27T17:14:30.430181281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"1","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430211426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"client":"2","":"downstairs","session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430237762Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430251759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"6dfb4c78-1f8d-4115-8656-c68fc9dbce82 is now active with session: a0a067cf-c8a4-4d19-a3f3-7b4c0abea324","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430264817Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.430278115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654,"session_id":"a0a067cf-c8a4-4d19-a3f3-7b4c0abea324"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:30.43030357Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 63 len 12288 data: 10 9 10 02/30 Flush 03/30 Write block 85 len 20480 data: 11 10 10 8 8 04/30 Read block 426 len 28672 05/30 Flush 06/30 Write block 168 len 28672 data: 14 11 9 10 9 8 9 07/30 Read block 364 len 12288 08/30 Write block 460 len 28672 data: 9 10 11 13 13 10 8 09/30 Read block 177 len 24576 10/30 Read block 6 len 8192 11/30 Write block 328 len 28672 data: 9 8 7 10 10 11 12 12/30 Flush 13/30 Write block 364 len 20480 data: 16 14 14 14 12 14/30 Read block 249 len 20480 15/30 Read block 242 len 24576 16/30 Read block 493 len 8192 17/30 Write block 202 len 40960 data: 11 10 11 7 7 8 8 11 10 8 18/30 Read block 296 len 12288 19/30 Write block 258 len 28672 data: 7 8 9 9 8 8 7 20/30 Write block 19 len 20480 data: 13 8 8 6 6 21/30 Read block 430 len 20480 22/30 Write block 456 len 36864 data: 14 13 12 9 10 11 12 14 14 23/30 Read block 250 len 36864 24/30 Read block 29 len 40960 25/30 Read block 483 len 4096 26/30 Write block 412 len 36864 data: 10 10 10 12 10 10 7 8 7 27/30 Read block 581 len 12288 28/30 Read block 454 len 16384 29/30 Read block 547 len 24576 30/30 Read block 449 len 4096 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 5 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Read 6 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:30.462838458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 195 195 195 T T T 1 020-039 116 116 116 205 205 205 T T T 2 040-059 113 113 113 198 198 198 F F F 3 060-079 118 118 118 207 207 207 F F F 4 080-099 118 118 118 208 208 208 F F F 5 100-119 115 115 115 202 202 202 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 118 118 118 209 209 209 F F F 9 180-199 117 117 117 206 206 206 F F F 10 200-219 104 104 104 185 185 185 T T T 11 220-239 117 117 117 206 206 206 F F F 12 240-259 117 117 117 206 206 206 T T T 13 260-279 107 107 107 190 190 190 T T T 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 117 117 117 206 206 206 T T T 19 380-399 110 110 110 194 194 194 F F F 20 400-419 113 113 113 198 198 198 T T T 21 420-439 116 116 116 205 205 205 T T T 22 440-459 115 115 115 202 202 202 T T T 23 460-479 118 118 118 209 209 209 T T T 24 480-499 112 112 112 196 196 196 F F F 25 500-519 117 117 117 206 206 206 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 118, Max flush: 209 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 210 210 210 F F F 1 020-039 119 119 119 210 210 210 F F F 2 040-059 113 113 113 198 198 198 F F F 3 060-079 118 118 118 207 207 207 F F F 4 080-099 118 118 118 208 208 208 F F F 5 100-119 115 115 115 202 202 202 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 118 118 118 209 209 209 F F F 9 180-199 117 117 117 206 206 206 F F F 10 200-219 119 119 119 210 210 210 F F F 11 220-239 117 117 117 206 206 206 F F F 12 240-259 119 119 119 210 210 210 F F F 13 260-279 119 119 119 210 210 210 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 119 119 119 210 210 210 F F F 19 380-399 110 110 110 194 194 194 F F F 20 400-419 119 119 119 210 210 210 F F F 21 420-439 119 119 119 210 210 210 F F F 22 440-459 119 119 119 210 210 210 F F F 23 460-479 119 119 119 210 210 210 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 117 117 117 206 206 206 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 104 104 104 185 185 185 F F F 28 560-579 109 109 109 192 192 192 F F F 29 580-599 112 112 112 197 197 197 F F F Max gen: 119, Max flush: 210 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.861674133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.906601574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.907468124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.907505327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.907525194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.907540091Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662} {"msg":"Crucible 2adcca59-258a-4e95-87a7-5c61e1c38639 has session id: 04e42288-6025-4012-acd1-3533862e4bf4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.907996622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Upstairs opts: Upstairs UUID: 2adcca59-258a-4e95-87a7-5c61e1c38639, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.908030346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Crucible stats registered with UUID: 2adcca59-258a-4e95-87a7-5c61e1c38639","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.908045583Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939137073Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662} {"msg":"2adcca59-258a-4e95-87a7-5c61e1c38639 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939190203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939222278Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.93926519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939302494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939340227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939378041Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939412975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939437171Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939478873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939518266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49148)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.93955525Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38350)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939597503Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42865)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.939637196Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.940391974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.940428418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.940446725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.940462702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.940628793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.940663257Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941154581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941262773Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941412247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941716514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941758396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941783632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941805978Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.941829234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.94185338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942033788Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942061264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[0]R flush_numbers[0..12]: [210, 210, 198, 207, 208, 202, 201, 203, 209, 206, 210, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942077141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[0]R generation[0..12]: [119, 119, 113, 118, 118, 115, 114, 116, 118, 117, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942091629Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942105986Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[1]R flush_numbers[0..12]: [210, 210, 198, 207, 208, 202, 201, 203, 209, 206, 210, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942119924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[1]R generation[0..12]: [119, 119, 113, 118, 118, 115, 114, 116, 118, 117, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942134231Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942148748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[2]R flush_numbers[0..12]: [210, 210, 198, 207, 208, 202, 201, 203, 209, 206, 210, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942162526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[2]R generation[0..12]: [119, 119, 113, 118, 118, 115, 114, 116, 118, 117, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942179703Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942193881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942206929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942220006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942232824Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942245652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"":"downstairs","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942258969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Transition from WaitQuorum to Active","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942281425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"0","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942294873Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"1","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942308171Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"client":"2","":"downstairs","session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942321629Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942333846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"2adcca59-258a-4e95-87a7-5c61e1c38639 is now active with session: 04e42288-6025-4012-acd1-3533862e4bf4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942346544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.942358522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662,"session_id":"04e42288-6025-4012-acd1-3533862e4bf4"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:34.94237078Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 33 len 16384 data: 9 11 9 8 02/30 Read block 54 len 8192 03/30 Write block 453 len 32768 data: 11 13 12 15 14 13 10 11 04/30 Read block 161 len 8192 05/30 Read block 14 len 8192 06/30 Flush 07/30 Write block 555 len 36864 data: 10 11 11 8 8 8 9 9 7 08/30 Read block 507 len 4096 09/30 Flush 10/30 Flush 11/30 Write block 109 len 28672 data: 8 9 8 9 6 8 5 12/30 Read block 64 len 20480 13/30 Flush 14/30 Write block 206 len 28672 data: 8 9 9 12 11 9 6 15/30 Read block 370 len 32768 16/30 Read block 260 len 40960 17/30 Write block 367 len 4096 data: 15 18/30 Read block 222 len 24576 19/30 Read block 156 len 40960 20/30 Write block 385 len 32768 data: 9 10 9 8 8 8 9 7 21/30 Flush 22/30 Flush 23/30 Write block 13 len 28672 data: 15 11 13 13 13 13 14 24/30 Write block 582 len 16384 data: 9 11 7 8 25/30 Write block 417 len 32768 data: 11 8 9 8 8 9 7 8 26/30 Flush 27/30 Write block 43 len 8192 data: 9 9 28/30 Read block 213 len 28672 29/30 Write block 304 len 4096 data: 7 30/30 Read block 16 len 40960 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:34.971172308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 217 217 217 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 113 113 113 198 198 198 T T T 3 060-079 118 118 118 207 207 207 F F F 4 080-099 118 118 118 208 208 208 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 118 118 118 209 209 209 F F F 9 180-199 117 117 117 206 206 206 F F F 10 200-219 120 120 120 215 215 215 F F F 11 220-239 117 117 117 206 206 206 F F F 12 240-259 119 119 119 210 210 210 F F F 13 260-279 119 119 119 210 210 210 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 112 112 112 196 196 196 T T T 16 320-339 118 118 118 209 209 209 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 120 120 120 215 215 215 F F F 19 380-399 120 120 120 215 215 215 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 117 117 117 206 206 206 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 120 120 120 212 212 212 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 120 120 120 217 217 217 F F F Max gen: 120, Max flush: 217 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 217 217 217 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 121 121 121 218 218 218 F F F 3 060-079 118 118 118 207 207 207 F F F 4 080-099 118 118 118 208 208 208 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 114 114 114 201 201 201 F F F 7 140-159 116 116 116 203 203 203 F F F 8 160-179 118 118 118 209 209 209 F F F 9 180-199 117 117 117 206 206 206 F F F 10 200-219 120 120 120 215 215 215 F F F 11 220-239 117 117 117 206 206 206 F F F 12 240-259 119 119 119 210 210 210 F F F 13 260-279 119 119 119 210 210 210 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 121 121 121 218 218 218 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 120 120 120 215 215 215 F F F 19 380-399 120 120 120 215 215 215 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 112 112 112 196 196 196 F F F 25 500-519 117 117 117 206 206 206 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 120 120 120 212 212 212 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 120 120 120 217 217 217 F F F Max gen: 121, Max flush: 218 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.248404884Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:39.292804158Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.293353382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.325649252Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.326632461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.326669625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.326684462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.326698609Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670} {"msg":"Crucible e9a8d0bf-6467-44aa-8b3e-dc60ae80253f has session id: bed3348c-fa85-44aa-ae51-acb232e0eb1c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.327183435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Upstairs opts: Upstairs UUID: e9a8d0bf-6467-44aa-8b3e-dc60ae80253f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.327228647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Crucible stats registered with UUID: e9a8d0bf-6467-44aa-8b3e-dc60ae80253f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.327253863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.358899556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670} {"msg":"e9a8d0bf-6467-44aa-8b3e-dc60ae80253f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.358948707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.358964894Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.358987661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359014316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359041421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359056889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359070736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359129396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359217361Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359256494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51974)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359286849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63278)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359323292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38229)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.359587096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360136911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360172764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360201759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360229725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360316769Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360341645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360819682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.360849986Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361035514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361425986Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361449592Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.3614637Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361529348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361552704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361566662Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361593357Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361607065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[0]R flush_numbers[0..12]: [217, 211, 218, 207, 208, 214, 201, 203, 209, 206, 215, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361673513Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[0]R generation[0..12]: [120, 120, 121, 118, 118, 120, 114, 116, 118, 117, 120, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.36169391Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361708317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[1]R flush_numbers[0..12]: [217, 211, 218, 207, 208, 214, 201, 203, 209, 206, 215, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361722185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[1]R generation[0..12]: [120, 120, 121, 118, 118, 120, 114, 116, 118, 117, 120, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361735922Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.36174961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[2]R flush_numbers[0..12]: [217, 211, 218, 207, 208, 214, 201, 203, 209, 206, 215, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361763008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[2]R generation[0..12]: [120, 120, 121, 118, 118, 120, 114, 116, 118, 117, 120, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361776495Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361790133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361803091Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361819758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361832726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361845603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361878638Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361892425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"0","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361906063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"1","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361919271Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"client":"2","":"downstairs","session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361932648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361945006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"e9a8d0bf-6467-44aa-8b3e-dc60ae80253f is now active with session: bed3348c-fa85-44aa-ae51-acb232e0eb1c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361957624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361969522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670,"session_id":"bed3348c-fa85-44aa-ae51-acb232e0eb1c"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:39.361981689Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 32768 data: 10 8 7 6 7 8 8 7 02/30 Read block 178 len 24576 03/30 Read block 493 len 36864 04/30 Read block 462 len 20480 05/30 Write block 201 len 8192 data: 11 12 06/30 Read block 583 len 32768 07/30 Write block 582 len 16384 data: 10 12 8 9 08/30 Read block 441 len 12288 09/30 Write block 178 len 36864 data: 7 7 9 7 9 9 9 10 10 10/30 Write block 65 len 20480 data: 11 13 11 11 12 11/30 Write block 377 len 40960 data: 9 8 8 7 10 10 12 10 10 11 12/30 Write block 541 len 4096 data: 12 13/30 Read block 500 len 40960 14/30 Read block 297 len 4096 15/30 Read block 130 len 16384 16/30 Write block 9 len 40960 data: 9 12 12 14 16 12 14 14 14 14 17/30 Write block 500 len 12288 data: 7 9 10 18/30 Read block 90 len 32768 19/30 Read block 508 len 40960 20/30 Read block 486 len 40960 21/30 Read block 21 len 20480 22/30 Read block 119 len 8192 23/30 Write block 556 len 36864 data: 12 12 9 9 9 10 10 8 8 24/30 Write block 589 len 24576 data: 7 9 10 12 12 11 25/30 Read block 555 len 40960 26/30 Write block 124 len 8192 data: 14 11 27/30 Write block 332 len 4096 data: 11 28/30 Write block 269 len 40960 data: 6 5 6 6 6 7 7 9 8 8 29/30 Read block 354 len 24576 30/30 Write block 482 len 32768 data: 7 9 9 9 9 8 6 7 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 8 Done Done Done false 2 Acked 1001 Read 6 Done Done Done false 3 Acked 1002 Read 9 Done Done Done false 4 Acked 1003 Read 5 Done Done Done false 5 Acked 1004 Write 2 Done Done Done false 6 Acked 1005 Read 8 Done Done Done false 7 Acked 1006 Write 4 Done Done Done false 8 Acked 1007 Read 3 Done Done Done false 9 Acked 1008 Write 9 Done Done Done false 10 Acked 1009 Write 5 Done Done Done false 11 Acked 1010 Write 10 Done Done Done false 12 Acked 1011 Write 1 Done Done Done false 13 Acked 1012 Read 10 Done Done Done false 14 Acked 1013 Read 1 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Write 10 Done Done Done false 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:39.383099895Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 217 217 217 T T T 1 020-039 120 120 120 211 211 211 F F F 2 040-059 121 121 121 218 218 218 F F F 3 060-079 118 118 118 207 207 207 T T T 4 080-099 118 118 118 208 208 208 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 114 114 114 201 201 201 T T T 7 140-159 116 116 116 203 203 203 T T T 8 160-179 118 118 118 209 209 209 T T T 9 180-199 117 117 117 206 206 206 T T T 10 200-219 120 120 120 215 215 215 T T T 11 220-239 117 117 117 206 206 206 F F F 12 240-259 119 119 119 210 210 210 F F F 13 260-279 119 119 119 210 210 210 T T T 14 280-299 107 107 107 190 190 190 F F F 15 300-319 121 121 121 218 218 218 F F F 16 320-339 118 118 118 209 209 209 T T T 17 340-359 114 114 114 200 200 200 F F F 18 360-379 120 120 120 215 215 215 T T T 19 380-399 120 120 120 215 215 215 T T T 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 112 112 112 196 196 196 T T T 25 500-519 117 117 117 206 206 206 T T T 26 520-539 114 114 114 199 199 199 F F F 27 540-559 120 120 120 212 212 212 T T T 28 560-579 120 120 120 212 212 212 T T T 29 580-599 120 120 120 217 217 217 T T T Max gen: 121, Max flush: 218 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 121 121 121 218 218 218 F F F 3 060-079 122 122 122 219 219 219 F F F 4 080-099 118 118 118 208 208 208 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 122 122 122 219 219 219 F F F 9 180-199 122 122 122 219 219 219 F F F 10 200-219 122 122 122 219 219 219 F F F 11 220-239 117 117 117 206 206 206 F F F 12 240-259 119 119 119 210 210 210 F F F 13 260-279 122 122 122 219 219 219 F F F 14 280-299 107 107 107 190 190 190 F F F 15 300-319 121 121 121 218 218 218 F F F 16 320-339 122 122 122 219 219 219 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 122 122 122 219 219 219 F F F 19 380-399 122 122 122 219 219 219 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 122 122 122 219 219 219 F F F 25 500-519 122 122 122 219 219 219 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 122 122 122 219 219 219 F F F 28 560-579 122 122 122 219 219 219 F F F 29 580-599 122 122 122 219 219 219 F F F Max gen: 122, Max flush: 219 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.881129677Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:43.923011699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.923386364Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.954769052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.955555206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.95558858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.955607126Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.955621884Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678} {"msg":"Crucible 40ac09bf-0e16-4229-9d97-771c287e0902 has session id: 95de6653-38f8-44f8-84e6-a09df700fb69","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.956136954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Upstairs opts: Upstairs UUID: 40ac09bf-0e16-4229-9d97-771c287e0902, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.956197564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Crucible stats registered with UUID: 40ac09bf-0e16-4229-9d97-771c287e0902","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.956225709Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988026845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678} {"msg":"40ac09bf-0e16-4229-9d97-771c287e0902 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988080006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988099162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988129767Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988147884Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988165161Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988181758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988198545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988221271Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988249546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.98828317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_connection connected from Ok(127.0.0.1:41623)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988327103Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_connection connected from Ok(127.0.0.1:64658)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988366356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_connection connected from Ok(127.0.0.1:45744)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.988650256Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.989368871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.989414733Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.989444218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.989467074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.989532753Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.989552229Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990016289Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990171222Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990322515Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990654978Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990680713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990695351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990782945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990803382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.990818039Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991017674Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991037151Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[0]R flush_numbers[0..12]: [219, 211, 218, 219, 208, 214, 219, 219, 219, 219, 219, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991063636Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[0]R generation[0..12]: [122, 120, 121, 122, 118, 120, 122, 122, 122, 122, 122, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991078104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991092151Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[1]R flush_numbers[0..12]: [219, 211, 218, 219, 208, 214, 219, 219, 219, 219, 219, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991105909Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[1]R generation[0..12]: [122, 120, 121, 122, 118, 120, 122, 122, 122, 122, 122, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991119377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991133514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[2]R flush_numbers[0..12]: [219, 211, 218, 219, 208, 214, 219, 219, 219, 219, 219, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991147532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[2]R generation[0..12]: [122, 120, 121, 122, 118, 120, 122, 122, 122, 122, 122, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991164409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991178516Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Max found gen is 123","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991191314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Generation requested: 124 >= found:123","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991204232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.99121704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991229967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"":"downstairs","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991243025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991254853Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"0","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991283658Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"1","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991309234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"client":"2","":"downstairs","session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991335269Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991358555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"40ac09bf-0e16-4229-9d97-771c287e0902 is now active with session: 95de6653-38f8-44f8-84e6-a09df700fb69","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991382701Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:14:43.991397268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678,"session_id":"95de6653-38f8-44f8-84e6-a09df700fb69"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:43.991412426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 20480 data: 11 11 9 9 7 02/30 Read block 226 len 12288 03/30 Read block 521 len 36864 04/30 Read block 335 len 28672 05/30 Write block 169 len 12288 data: 12 10 11 06/30 Read block 185 len 4096 07/30 Write block 483 len 12288 data: 10 10 10 08/30 Read block 349 len 24576 09/30 Write block 169 len 24576 data: 13 11 12 10 9 10 10/30 Write block 200 len 20480 data: 11 12 13 11 12 11/30 Write block 76 len 32768 data: 13 8 8 11 10 9 10 11 12/30 Read block 292 len 40960 13/30 Write block 192 len 32768 data: 9 6 7 9 11 11 11 11 14/30 Write block 175 len 40960 data: 8 8 8 8 8 10 8 10 10 10 15/30 Read block 537 len 16384 16/30 Read block 577 len 8192 17/30 Read block 406 len 32768 18/30 Read block 572 len 40960 19/30 Read block 435 len 40960 20/30 Read block 195 len 28672 21/30 Read block 57 len 12288 22/30 Read block 84 len 20480 23/30 Read block 263 len 4096 24/30 Read block 346 len 24576 25/30 Read block 416 len 16384 26/30 Write block 484 len 24576 data: 11 11 10 9 7 8 27/30 Write block 512 len 16384 data: 10 10 12 14 28/30 Write block 556 len 16384 data: 13 13 10 10 29/30 Read block 409 len 12288 30/30 Write block 296 len 12288 data: 4 5 4 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 5 Done Done Done false 2 Acked 1001 Read 3 Done Done Done false 3 Acked 1002 Read 9 Done Done Done false 4 Acked 1003 Read 7 Done Done Done false 5 Acked 1004 Write 3 Done Done Done false 6 Acked 1005 Read 1 Done Done Done false 7 Acked 1006 Write 3 Done Done Done false 8 Acked 1007 Read 6 Done Done Done false 9 Acked 1008 Write 6 Done Done Done false 10 Acked 1009 Write 5 Done Done Done false 11 Acked 1010 Write 8 Done Done Done false 12 Acked 1011 Read 10 Done Done Done false 13 Acked 1012 Write 8 Done Done Done false 14 Acked 1013 Write 10 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Read 8 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:44.01224522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 121 121 121 218 218 218 F F F 3 060-079 122 122 122 219 219 219 T T T 4 080-099 118 118 118 208 208 208 T T T 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 122 122 122 219 219 219 T T T 9 180-199 122 122 122 219 219 219 T T T 10 200-219 122 122 122 219 219 219 T T T 11 220-239 117 117 117 206 206 206 F F F 12 240-259 119 119 119 210 210 210 T T T 13 260-279 122 122 122 219 219 219 F F F 14 280-299 107 107 107 190 190 190 T T T 15 300-319 121 121 121 218 218 218 F F F 16 320-339 122 122 122 219 219 219 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 122 122 122 219 219 219 F F F 19 380-399 122 122 122 219 219 219 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 122 122 122 219 219 219 T T T 25 500-519 122 122 122 219 219 219 T T T 26 520-539 114 114 114 199 199 199 F F F 27 540-559 122 122 122 219 219 219 T T T 28 560-579 122 122 122 219 219 219 F F F 29 580-599 122 122 122 219 219 219 F F F Max gen: 122, Max flush: 219 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 121 121 121 218 218 218 F F F 3 060-079 123 123 123 220 220 220 F F F 4 080-099 123 123 123 220 220 220 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 123 123 123 220 220 220 F F F 10 200-219 123 123 123 220 220 220 F F F 11 220-239 117 117 117 206 206 206 F F F 12 240-259 123 123 123 220 220 220 F F F 13 260-279 122 122 122 219 219 219 F F F 14 280-299 123 123 123 220 220 220 F F F 15 300-319 121 121 121 218 218 218 F F F 16 320-339 122 122 122 219 219 219 F F F 17 340-359 114 114 114 200 200 200 F F F 18 360-379 122 122 122 219 219 219 F F F 19 380-399 122 122 122 219 219 219 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 120 120 120 211 211 211 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 123 123 123 220 220 220 F F F 25 500-519 123 123 123 220 220 220 F F F 26 520-539 114 114 114 199 199 199 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 122 122 122 219 219 219 F F F 29 580-599 122 122 122 219 219 219 F F F Max gen: 123, Max flush: 220 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.437135271Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.48196486Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.482898608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.482935331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.482949659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.482963686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686} {"msg":"Crucible 63d146bd-de62-4b11-bea0-7fb1b0c1c020 has session id: 1ed9f3d6-0b6d-413e-b666-cef03585c31c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.483513231Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Upstairs opts: Upstairs UUID: 63d146bd-de62-4b11-bea0-7fb1b0c1c020, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.483560743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Crucible stats registered with UUID: 63d146bd-de62-4b11-bea0-7fb1b0c1c020","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.483584438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514586883Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686} {"msg":"63d146bd-de62-4b11-bea0-7fb1b0c1c020 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514643003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514658501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514682347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514696844Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514710782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514733158Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514764622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514792428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514819453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514847878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40370)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.514983214Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59081)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.515046263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41054)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.515077038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.515884927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.515921051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.515950056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.516017204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.516051968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.516080793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.516612341Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.516655013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.516835852Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.51725142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517281224Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517296762Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517309499Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517323467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517336535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517505545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.51753419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[0]R flush_numbers[0..12]: [219, 211, 218, 220, 220, 214, 219, 219, 220, 220, 220, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517550308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[0]R generation[0..12]: [122, 120, 121, 123, 123, 120, 122, 122, 123, 123, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517564975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517579372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[1]R flush_numbers[0..12]: [219, 211, 218, 220, 220, 214, 219, 219, 220, 220, 220, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.51759381Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[1]R generation[0..12]: [122, 120, 121, 123, 123, 120, 122, 122, 123, 123, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517607578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517624745Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[2]R flush_numbers[0..12]: [219, 211, 218, 220, 220, 214, 219, 219, 220, 220, 220, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517638732Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[2]R generation[0..12]: [122, 120, 121, 123, 123, 120, 122, 122, 123, 123, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.51765246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517666707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517679855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Generation requested: 126 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517693053Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.51770585Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517718578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"":"downstairs","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517731576Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517743554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"0","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517756662Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"1","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"}Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517789996Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"client":"2","":"downstairs","session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517803524Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517815741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"63d146bd-de62-4b11-bea0-7fb1b0c1c020 is now active with session: 1ed9f3d6-0b6d-413e-b666-cef03585c31c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517828539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517842367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686,"session_id":"1ed9f3d6-0b6d-413e-b666-cef03585c31c"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:48.517854835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 590 len 12288 data: 10 11 13 02/30 Write block 240 len 28672 data: 12 13 12 12 10 10 8 03/30 Write block 335 len 20480 data: 16 15 15 17 15 04/30 Write block 342 len 28672 data: 15 13 11 6 7 5 6 05/30 Read block 66 len 20480 06/30 Read block 329 len 24576 07/30 Write block 232 len 20480 data: 7 9 10 12 11 08/30 Read block 543 len 40960 09/30 Write block 334 len 20480 data: 13 17 16 16 18 10/30 Read block 330 len 12288 11/30 Read block 135 len 24576 12/30 Read block 110 len 36864 13/30 Write block 487 len 32768 data: 10 8 9 6 7 5 4 6 14/30 Flush 15/30 Write block 329 len 40960 data: 9 8 11 12 12 14 18 17 17 19 16/30 Read block 477 len 4096 17/30 Read block 266 len 28672 18/30 Write block 24 len 8192 data: 7 7 19/30 Read block 261 len 20480 20/30 Write block 282 len 8192 data: 9 9 21/30 Read block 339 len 32768 22/30 Write block 528 len 32768 data: 6 11 12 9 9 10 9 11 23/30 Write block 311 len 8192 data: 12 10 24/30 Read block 440 len 16384 25/30 Read block 476 len 20480 26/30 Write block 231 len 16384 data: 8 8 10 11 27/30 Write block 443 len 24576 data: 6 7 10 13 13 11 28/30 Read block 193 len 8192 29/30 Read block 130 len 28672 30/30 Read block 579 len 12288 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 15 Acked 1014 Write 10 Done Done Done false 16 Acked 1015 Read 1 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:48.541459407Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 120 120 120 211 211 211 T T T 2 040-059 121 121 121 218 218 218 F F F 3 060-079 123 123 123 220 220 220 F F F 4 080-099 123 123 123 220 220 220 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 123 123 123 220 220 220 F F F 10 200-219 123 123 123 220 220 220 F F F 11 220-239 126 126 126 221 221 221 T T T 12 240-259 126 126 126 221 221 221 F F F 13 260-279 122 122 122 219 219 219 F F F 14 280-299 123 123 123 220 220 220 T T T 15 300-319 121 121 121 218 218 218 T T T 16 320-339 126 126 126 221 221 221 T T T 17 340-359 126 126 126 221 221 221 F F F 18 360-379 122 122 122 219 219 219 F F F 19 380-399 122 122 122 219 219 219 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 120 120 120 211 211 211 T T T 23 460-479 120 120 120 211 211 211 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 123 123 123 220 220 220 F F F 26 520-539 114 114 114 199 199 199 T T T 27 540-559 123 123 123 220 220 220 F F F 28 560-579 122 122 122 219 219 219 F F F 29 580-599 126 126 126 221 221 221 F F F Max gen: 126, Max flush: 221 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 127 127 127 222 222 222 F F F 2 040-059 121 121 121 218 218 218 F F F 3 060-079 123 123 123 220 220 220 F F F 4 080-099 123 123 123 220 220 220 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 123 123 123 220 220 220 F F F 10 200-219 123 123 123 220 220 220 F F F 11 220-239 127 127 127 222 222 222 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 122 122 122 219 219 219 F F F 14 280-299 127 127 127 222 222 222 F F F 15 300-319 127 127 127 222 222 222 F F F 16 320-339 127 127 127 222 222 222 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 122 122 122 219 219 219 F F F 19 380-399 122 122 122 219 219 219 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 F F F 22 440-459 127 127 127 222 222 222 F F F 23 460-479 120 120 120 211 211 211 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 123 123 123 220 220 220 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 122 122 122 219 219 219 F F F 29 580-599 126 126 126 221 221 221 F F F Max gen: 127, Max flush: 222 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.890791736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.935009911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.935944978Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.935982812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.936002588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.936017366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694} {"msg":"Crucible 3f3b16d3-1c19-43ae-9a6c-f1526385c765 has session id: c047d37e-6701-480e-81b6-e61e5c00df12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.936519308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Upstairs opts: Upstairs UUID: 3f3b16d3-1c19-43ae-9a6c-f1526385c765, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.93656761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Crucible stats registered with UUID: 3f3b16d3-1c19-43ae-9a6c-f1526385c765","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.936587736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968118719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694} {"msg":"3f3b16d3-1c19-43ae-9a6c-f1526385c765 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968184298Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968216462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968260485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968295878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968329453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968364757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968388812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968411938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968429945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.96845748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_connection connected from Ok(127.0.0.1:48644)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968492804Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_connection connected from Ok(127.0.0.1:34083)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968528168Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_connection connected from Ok(127.0.0.1:42838)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.968564862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.969287096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.969317631Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.969334048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.969348225Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.969583624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.969613479Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970121311Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970151186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970290981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970796963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970825348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970840296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970853553Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970866961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970880209Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970963634Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.970983031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[0]R flush_numbers[0..12]: [219, 222, 218, 220, 220, 214, 219, 219, 220, 220, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971012266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[0]R generation[0..12]: [122, 127, 121, 123, 123, 120, 122, 122, 123, 123, 123, 127]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971027743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.97104198Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[1]R flush_numbers[0..12]: [219, 222, 218, 220, 220, 214, 219, 219, 220, 220, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971055938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[1]R generation[0..12]: [122, 127, 121, 123, 123, 120, 122, 122, 123, 123, 123, 127]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971069386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971083203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[2]R flush_numbers[0..12]: [219, 222, 218, 220, 220, 214, 219, 219, 220, 220, 220, 222]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971096781Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[2]R generation[0..12]: [122, 127, 121, 123, 123, 120, 122, 122, 123, 123, 123, 127]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971116238Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971130595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971143463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971156811Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971169908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971182816Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"":"downstairs","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971196084Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.97121888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"0","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971232278Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"1","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971245155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"client":"2","":"downstairs","session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971258213Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971270381Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"3f3b16d3-1c19-43ae-9a6c-f1526385c765 is now active with session: c047d37e-6701-480e-81b6-e61e5c00df12","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971283269Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971295247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694,"session_id":"c047d37e-6701-480e-81b6-e61e5c00df12"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:52.971307334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 510 len 8192 data: 8 9 02/30 Read block 35 len 20480 03/30 Flush 04/30 Write block 371 len 12288 data: 15 15 13 05/30 Write block 297 len 32768 data: 6 5 3 4 4 5 6 8 06/30 Read block 317 len 24576 07/30 Flush 08/30 Write block 284 len 24576 data: 7 7 8 8 9 9 09/30 Read block 519 len 36864 10/30 Write block 51 len 8192 data: 8 8 11/30 Write block 41 len 12288 data: 11 9 10 12/30 Read block 171 len 20480 13/30 Flush 14/30 Write block 460 len 4096 data: 12 15/30 Flush 16/30 Write block 325 len 40960 data: 6 6 8 10 10 9 12 13 13 15 17/30 Read block 529 len 4096 18/30 Read block 397 len 40960 19/30 Flush 20/30 Write block 270 len 32768 data: 6 7 7 7 8 8 10 9 21/30 Read block 105 len 36864 22/30 Write block 366 len 8192 data: 15 16 23/30 Flush 24/30 Write block 420 len 8192 data: 9 9 25/30 Write block 388 len 8192 data: 9 9 26/30 Read block 88 len 12288 27/30 Read block 98 len 4096 28/30 Read block 128 len 20480 29/30 Write block 467 len 8192 data: 8 9 30/30 Write block 214 len 40960 data: 8 6 8 8 9 11 7 7 7 6 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:52.999518925Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 127 127 127 222 222 222 F F F 2 040-059 128 128 128 225 225 225 F F F 3 060-079 123 123 123 220 220 220 F F F 4 080-099 123 123 123 220 220 220 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 123 123 123 220 220 220 F F F 10 200-219 123 123 123 220 220 220 T T T 11 220-239 127 127 127 222 222 222 T T T 12 240-259 126 126 126 221 221 221 F F F 13 260-279 128 128 128 228 228 228 F F F 14 280-299 128 128 128 225 225 225 F F F 15 300-319 128 128 128 224 224 224 F F F 16 320-339 128 128 128 227 227 227 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 128 128 128 228 228 228 F F F 19 380-399 122 122 122 219 219 219 T T T 20 400-419 120 120 120 217 217 217 F F F 21 420-439 120 120 120 217 217 217 T T T 22 440-459 127 127 127 222 222 222 F F F 23 460-479 128 128 128 226 226 226 T T T 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 223 223 223 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 122 122 122 219 219 219 F F F 29 580-599 126 126 126 221 221 221 F F F Max gen: 128, Max flush: 228 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 127 127 127 222 222 222 F F F 2 040-059 128 128 128 225 225 225 F F F 3 060-079 123 123 123 220 220 220 F F F 4 080-099 123 123 123 220 220 220 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 123 123 123 220 220 220 F F F 10 200-219 129 129 129 229 229 229 F F F 11 220-239 129 129 129 229 229 229 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 128 128 128 228 228 228 F F F 14 280-299 128 128 128 225 225 225 F F F 15 300-319 128 128 128 224 224 224 F F F 16 320-339 128 128 128 227 227 227 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 128 128 128 228 228 228 F F F 19 380-399 129 129 129 229 229 229 F F F 20 400-419 120 120 120 217 217 217 F F F 21 420-439 129 129 129 229 229 229 F F F 22 440-459 127 127 127 222 222 222 F F F 23 460-479 129 129 129 229 229 229 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 223 223 223 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 122 122 122 219 219 219 F F F 29 580-599 126 126 126 221 221 221 F F F Max gen: 129, Max flush: 229 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.322322141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.367361603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.368360939Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.368400383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.36841487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.368429327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702} {"msg":"Crucible 156ff47f-41ff-4122-ba17-b63a8816f023 has session id: dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.368829458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Upstairs opts: Upstairs UUID: 156ff47f-41ff-4122-ba17-b63a8816f023, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.3688743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Crucible stats registered with UUID: 156ff47f-41ff-4122-ba17-b63a8816f023","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.368898896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400266757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702} {"msg":"156ff47f-41ff-4122-ba17-b63a8816f023 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400323777Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400343554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400372289Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400390026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400407143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.40042365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400440597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400457214Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.40048356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400515624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_connection connected from Ok(127.0.0.1:43103)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400665048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_connection connected from Ok(127.0.0.1:62185)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400724558Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_connection connected from Ok(127.0.0.1:56872)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.400761281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.401482316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.401523448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.401555233Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.401584178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.401610793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.401626021Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402193972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402235305Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402268199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402900359Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402926124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402940602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.40295345Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402967127Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402984054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.402996592Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.40300997Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[0]R flush_numbers[0..12]: [219, 222, 225, 220, 220, 214, 219, 219, 220, 220, 229, 229]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403024058Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[0]R generation[0..12]: [122, 127, 128, 123, 123, 120, 122, 122, 123, 123, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403037915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403051783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[1]R flush_numbers[0..12]: [219, 222, 225, 220, 220, 214, 219, 219, 220, 220, 229, 229]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.40306541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[1]R generation[0..12]: [122, 127, 128, 123, 123, 120, 122, 122, 123, 123, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403078968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403092815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[2]R flush_numbers[0..12]: [219, 222, 225, 220, 220, 214, 219, 219, 220, 220, 229, 229]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403106283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[2]R generation[0..12]: [122, 127, 128, 123, 123, 120, 122, 122, 123, 123, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403120981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403135208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403148316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403161574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403176011Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403188949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403212874Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403224792Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"0","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.40323804Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"1","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403251508Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"client":"2","":"downstairs","session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403264825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403277083Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"156ff47f-41ff-4122-ba17-b63a8816f023 is now active with session: dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403289801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403301679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702,"session_id":"dee8c0f4-2c38-4a7c-8739-1b0bd26a1aea"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:14:57.403313717Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 304 len 12288 data: 9 8 12 02/30 Write block 596 len 4096 data: 8 03/30 Read block 42 len 20480 04/30 Read block 281 len 24576 05/30 Write block 343 len 4096 data: 14 06/30 Flush 07/30 Write block 306 len 32768 data: 13 12 11 11 13 13 11 7 08/30 Write block 387 len 24576 data: 10 10 10 9 10 8 09/30 Read block 181 len 32768 10/30 Write block 366 len 12288 data: 16 17 13 11/30 Write block 382 len 4096 data: 11 12/30 Write block 219 len 8192 data: 12 8 13/30 Write block 572 len 24576 data: 11 11 11 12 11 10 14/30 Read block 483 len 36864 15/30 Write block 404 len 28672 data: 13 13 14 14 13 10 10 16/30 Flush 17/30 Write block 506 len 12288 data: 8 10 10 18/30 Flush 19/30 Write block 181 len 32768 data: 9 11 11 11 11 11 10 9 20/30 Read block 388 len 12288 21/30 Read block 164 len 40960 22/30 Read block 544 len 40960 23/30 Flush 24/30 Write block 308 len 4096 data: 12 25/30 Write block 505 len 8192 data: 8 9 26/30 Read block 475 len 16384 27/30 Write block 259 len 24576 data: 9 10 10 9 9 8 28/30 Read block 311 len 36864 29/30 Read block 205 len 24576 30/30 Read block 99 len 40960 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:14:57.431897363Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 127 127 127 222 222 222 F F F 2 040-059 128 128 128 225 225 225 F F F 3 060-079 123 123 123 220 220 220 F F F 4 080-099 123 123 123 220 220 220 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 130 130 130 233 233 233 F F F 10 200-219 130 130 130 231 231 231 F F F 11 220-239 130 130 130 231 231 231 F F F 12 240-259 126 126 126 221 221 221 T T T 13 260-279 128 128 128 228 228 228 T T T 14 280-299 128 128 128 225 225 225 F F F 15 300-319 130 130 130 231 231 231 T T T 16 320-339 128 128 128 227 227 227 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 130 130 130 231 231 231 F F F 19 380-399 130 130 130 231 231 231 F F F 20 400-419 130 130 130 231 231 231 F F F 21 420-439 129 129 129 229 229 229 F F F 22 440-459 127 127 127 222 222 222 F F F 23 460-479 129 129 129 229 229 229 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 130 130 130 232 232 232 T T T 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 130 130 130 231 231 231 F F F 29 580-599 130 130 130 230 230 230 F F F Max gen: 130, Max flush: 233 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 127 127 127 222 222 222 F F F 2 040-059 128 128 128 225 225 225 F F F 3 060-079 123 123 123 220 220 220 F F F 4 080-099 123 123 123 220 220 220 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 122 122 122 219 219 219 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 130 130 130 233 233 233 F F F 10 200-219 130 130 130 231 231 231 F F F 11 220-239 130 130 130 231 231 231 F F F 12 240-259 131 131 131 234 234 234 F F F 13 260-279 131 131 131 234 234 234 F F F 14 280-299 128 128 128 225 225 225 F F F 15 300-319 131 131 131 234 234 234 F F F 16 320-339 128 128 128 227 227 227 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 130 130 130 231 231 231 F F F 19 380-399 130 130 130 231 231 231 F F F 20 400-419 130 130 130 231 231 231 F F F 21 420-439 129 129 129 229 229 229 F F F 22 440-459 127 127 127 222 222 222 F F F 23 460-479 129 129 129 229 229 229 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 131 131 131 234 234 234 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 130 130 130 231 231 231 F F F 29 580-599 130 130 130 230 230 230 F F F Max gen: 131, Max flush: 234 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.741498496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.785907888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.786798362Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.786845084Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.786862971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.786880818Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710} {"msg":"Crucible aa549532-4a93-4d03-9a95-1eecb44ec891 has session id: 9afe5323-b815-4b38-9e48-028d56c2860e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.787337209Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Upstairs opts: Upstairs UUID: aa549532-4a93-4d03-9a95-1eecb44ec891, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.787392649Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Crucible stats registered with UUID: aa549532-4a93-4d03-9a95-1eecb44ec891","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.787418335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818719727Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710} {"msg":"aa549532-4a93-4d03-9a95-1eecb44ec891 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818779137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818798314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818828099Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818846836Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818863942Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.81888075Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818897896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818914264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818931691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.818962645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38665)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.819141574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64270)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.819194095Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33236)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.81922571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.819951463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.819994666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.820024831Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.820055015Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.820265339Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.820294054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.820794917Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.820823952Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.820840909Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821474008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821503213Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821517841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821530178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821543406Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821556544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821568892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821582719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[0]R flush_numbers[0..12]: [219, 222, 225, 220, 220, 214, 219, 219, 220, 233, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821598797Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[0]R generation[0..12]: [122, 127, 128, 123, 123, 120, 122, 122, 123, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821615974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821630541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[1]R flush_numbers[0..12]: [219, 222, 225, 220, 220, 214, 219, 219, 220, 233, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821644559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[1]R generation[0..12]: [122, 127, 128, 123, 123, 120, 122, 122, 123, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821658206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821672044Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[2]R flush_numbers[0..12]: [219, 222, 225, 220, 220, 214, 219, 219, 220, 233, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821685611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[2]R generation[0..12]: [122, 127, 128, 123, 123, 120, 122, 122, 123, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821699029Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821712687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821725455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821738632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.82175152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821764248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"":"downstairs","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821777426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821789273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"0","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821816059Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"1","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821845114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"client":"2","":"downstairs","session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821860161Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821873939Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"aa549532-4a93-4d03-9a95-1eecb44ec891 is now active with session: 9afe5323-b815-4b38-9e48-028d56c2860e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821886896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821898714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710,"session_id":"9afe5323-b815-4b38-9e48-028d56c2860e"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:01.821910992Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 121 len 12288 data: 12 11 12 02/30 Write block 246 len 24576 data: 9 5 8 7 7 8 03/30 Read block 283 len 12288 04/30 Read block 237 len 24576 05/30 Read block 543 len 4096 06/30 Read block 406 len 40960 07/30 Read block 321 len 24576 08/30 Flush 09/30 Write block 485 len 4096 data: 12 10/30 Write block 260 len 12288 data: 11 11 10 11/30 Read block 530 len 4096 12/30 Read block 575 len 16384 13/30 Read block 492 len 12288 14/30 Write block 64 len 32768 data: 10 12 14 12 12 13 11 12 15/30 Read block 87 len 32768 16/30 Write block 31 len 24576 data: 9 8 10 12 10 9 17/30 Write block 475 len 40960 data: 13 9 7 9 9 11 9 8 11 12 18/30 Write block 464 len 16384 data: 15 11 9 9 19/30 Read block 221 len 16384 20/30 Read block 420 len 28672 21/30 Read block 266 len 16384 22/30 Write block 450 len 32768 data: 13 12 10 12 14 13 16 15 23/30 Read block 533 len 4096 24/30 Read block 262 len 40960 25/30 Write block 80 len 24576 data: 11 10 11 12 13 12 26/30 Write block 377 len 12288 data: 10 9 9 27/30 Read block 425 len 4096 28/30 Write block 88 len 40960 data: 9 9 6 5 5 5 9 9 11 11 29/30 Read block 192 len 40960 30/30 Read block 131 len 20480 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Write 3 Done Done Done false 11 Acked 1010 Read 1 Done Done Done false 12 Acked 1011 Read 4 Done Done Done false 13 Acked 1012 Read 3 Done Done Done false 14 Acked 1013 Write 8 Done Done Done false 15 Acked 1014 Read 8 Done Done Done false 16 Acked 1015 Write 6 Done Done Done false 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Read 4 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:01.844430433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 127 127 127 222 222 222 T T T 2 040-059 128 128 128 225 225 225 F F F 3 060-079 123 123 123 220 220 220 T T T 4 080-099 123 123 123 220 220 220 T T T 5 100-119 120 120 120 214 214 214 F F F 6 120-139 132 132 132 235 235 235 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 130 130 130 233 233 233 F F F 10 200-219 130 130 130 231 231 231 F F F 11 220-239 130 130 130 231 231 231 F F F 12 240-259 132 132 132 235 235 235 F F F 13 260-279 131 131 131 234 234 234 T T T 14 280-299 128 128 128 225 225 225 F F F 15 300-319 131 131 131 234 234 234 F F F 16 320-339 128 128 128 227 227 227 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 130 130 130 231 231 231 T T T 19 380-399 130 130 130 231 231 231 F F F 20 400-419 130 130 130 231 231 231 F F F 21 420-439 129 129 129 229 229 229 F F F 22 440-459 127 127 127 222 222 222 T T T 23 460-479 129 129 129 229 229 229 T T T 24 480-499 126 126 126 221 221 221 T T T 25 500-519 131 131 131 234 234 234 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 130 130 130 231 231 231 F F F 29 580-599 130 130 130 230 230 230 F F F Max gen: 132, Max flush: 235 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 128 128 128 225 225 225 F F F 3 060-079 133 133 133 236 236 236 F F F 4 080-099 133 133 133 236 236 236 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 132 132 132 235 235 235 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 F F F 9 180-199 130 130 130 233 233 233 F F F 10 200-219 130 130 130 231 231 231 F F F 11 220-239 130 130 130 231 231 231 F F F 12 240-259 132 132 132 235 235 235 F F F 13 260-279 133 133 133 236 236 236 F F F 14 280-299 128 128 128 225 225 225 F F F 15 300-319 131 131 131 234 234 234 F F F 16 320-339 128 128 128 227 227 227 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 133 133 133 236 236 236 F F F 19 380-399 130 130 130 231 231 231 F F F 20 400-419 130 130 130 231 231 231 F F F 21 420-439 129 129 129 229 229 229 F F F 22 440-459 133 133 133 236 236 236 F F F 23 460-479 133 133 133 236 236 236 F F F 24 480-499 133 133 133 236 236 236 F F F 25 500-519 131 131 131 234 234 234 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 130 130 130 231 231 231 F F F 29 580-599 130 130 130 230 230 230 F F F Max gen: 133, Max flush: 236 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.232269062Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:06.275478493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.276047104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.30853343Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.309519339Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.309560112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.309577259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.309595716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718} {"msg":"Crucible cc8ae087-ea39-4f2e-be3c-5cd0a2f1cc8e has session id: 49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.310040548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Upstairs opts: Upstairs UUID: cc8ae087-ea39-4f2e-be3c-5cd0a2f1cc8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.310095239Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Crucible stats registered with UUID: cc8ae087-ea39-4f2e-be3c-5cd0a2f1cc8e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.310131572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341513831Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718} {"msg":"cc8ae087-ea39-4f2e-be3c-5cd0a2f1cc8e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341563382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.34157938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341606445Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341621822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.3416387Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341655246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341671894Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341688051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341704828Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341721595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36607)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.34186727Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:59643)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.341910782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41161)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.34226979Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.342712123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.342745647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.342761274Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.3427844Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.342998483Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.343038676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.343554786Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.343585231Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.343601448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344160161Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344185656Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344200064Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344212721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344226149Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344239487Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344252105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344270082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[0]R flush_numbers[0..12]: [219, 236, 225, 236, 236, 214, 235, 219, 220, 233, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344297237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[0]R generation[0..12]: [122, 133, 128, 133, 133, 120, 132, 122, 123, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344311854Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344326072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[1]R flush_numbers[0..12]: [219, 236, 225, 236, 236, 214, 235, 219, 220, 233, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344339779Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[1]R generation[0..12]: [122, 133, 128, 133, 133, 120, 132, 122, 123, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344353417Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344367265Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[2]R flush_numbers[0..12]: [219, 236, 225, 236, 236, 214, 235, 219, 220, 233, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344381282Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[2]R generation[0..12]: [122, 133, 128, 133, 133, 120, 132, 122, 123, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.34439496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344408817Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344421695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344434773Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344447551Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344460358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"":"downstairs","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344475636Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344487784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"0","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344512409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"1","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344525737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"client":"2","":"downstairs","session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344538695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344550993Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"cc8ae087-ea39-4f2e-be3c-5cd0a2f1cc8e is now active with session: 49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.3445637Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344575788Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718,"session_id":"49a049fb-5c2a-4f9a-a192-74ccb2f4b1b4"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:06.344587816Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 261 len 12288 data: 12 11 10 02/30 Read block 360 len 20480 03/30 Flush 04/30 Flush 05/30 Write block 471 len 32768 data: 8 10 8 10 14 10 8 10 06/30 Flush 07/30 Write block 62 len 8192 data: 11 11 08/30 Read block 364 len 12288 09/30 Read block 446 len 36864 10/30 Read block 317 len 8192 11/30 Write block 45 len 4096 data: 9 12/30 Read block 445 len 32768 13/30 Write block 191 len 16384 data: 9 10 7 8 14/30 Read block 400 len 12288 15/30 Read block 477 len 4096 16/30 Write block 361 len 24576 data: 14 17 16 17 15 17 17/30 Read block 125 len 32768 18/30 Write block 294 len 32768 data: 6 4 5 7 6 4 5 5 19/30 Read block 539 len 32768 20/30 Write block 308 len 20480 data: 13 12 14 14 12 21/30 Read block 135 len 36864 22/30 Read block 242 len 8192 23/30 Read block 51 len 28672 24/30 Read block 545 len 8192 25/30 Write block 164 len 28672 data: 14 12 13 12 15 14 12 26/30 Read block 443 len 8192 27/30 Write block 579 len 8192 data: 12 11 28/30 Read block 95 len 8192 29/30 Read block 241 len 32768 30/30 Write block 247 len 16384 data: 6 9 8 8 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 7 Acked 1006 Write 2 Done Done Done false 8 Acked 1007 Read 3 Done Done Done false 9 Acked 1008 Read 9 Done Done Done false 10 Acked 1009 Read 2 Done Done Done false 11 Acked 1010 Write 1 Done Done Done false 12 Acked 1011 Read 8 Done Done Done false 13 Acked 1012 Write 4 Done Done Done false 14 Acked 1013 Read 3 Done Done Done false 15 Acked 1014 Read 1 Done Done Done false 16 Acked 1015 Write 6 Done Done Done false 17 Acked 1016 Read 8 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 9 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:06.367812615Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 128 128 128 225 225 225 T T T 3 060-079 133 133 133 236 236 236 T T T 4 080-099 133 133 133 236 236 236 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 132 132 132 235 235 235 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 123 123 123 220 220 220 T T T 9 180-199 130 130 130 233 233 233 T T T 10 200-219 130 130 130 231 231 231 F F F 11 220-239 130 130 130 231 231 231 F F F 12 240-259 132 132 132 235 235 235 T T T 13 260-279 134 134 134 237 237 237 F F F 14 280-299 128 128 128 225 225 225 T T T 15 300-319 131 131 131 234 234 234 T T T 16 320-339 128 128 128 227 227 227 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 133 133 133 236 236 236 T T T 19 380-399 130 130 130 231 231 231 F F F 20 400-419 130 130 130 231 231 231 F F F 21 420-439 129 129 129 229 229 229 F F F 22 440-459 133 133 133 236 236 236 F F F 23 460-479 134 134 134 239 239 239 F F F 24 480-499 133 133 133 236 236 236 F F F 25 500-519 131 131 131 234 234 234 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 130 130 130 231 231 231 T T T 29 580-599 130 130 130 230 230 230 T T T Max gen: 134, Max flush: 239 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 135 135 135 240 240 240 F F F 3 060-079 135 135 135 240 240 240 F F F 4 080-099 133 133 133 236 236 236 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 132 132 132 235 235 235 F F F 7 140-159 122 122 122 219 219 219 F F F 8 160-179 135 135 135 240 240 240 F F F 9 180-199 135 135 135 240 240 240 F F F 10 200-219 130 130 130 231 231 231 F F F 11 220-239 130 130 130 231 231 231 F F F 12 240-259 135 135 135 240 240 240 F F F 13 260-279 134 134 134 237 237 237 F F F 14 280-299 135 135 135 240 240 240 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 128 128 128 227 227 227 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 135 135 135 240 240 240 F F F 19 380-399 130 130 130 231 231 231 F F F 20 400-419 130 130 130 231 231 231 F F F 21 420-439 129 129 129 229 229 229 F F F 22 440-459 133 133 133 236 236 236 F F F 23 460-479 134 134 134 239 239 239 F F F 24 480-499 133 133 133 236 236 236 F F F 25 500-519 131 131 131 234 234 234 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 135 135 135 240 240 240 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 135, Max flush: 240 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.784730942Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.828509674Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.829296227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.829348478Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.829371544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.82939529Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726} {"msg":"Crucible 6d499d22-6278-4f0b-bf12-6ce4fc238a8f has session id: ee1569bc-cca9-4b4d-b6ee-0550357cb845","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.829836793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Upstairs opts: Upstairs UUID: 6d499d22-6278-4f0b-bf12-6ce4fc238a8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.829875586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Crucible stats registered with UUID: 6d499d22-6278-4f0b-bf12-6ce4fc238a8f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.829897033Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.860990831Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726} {"msg":"6d499d22-6278-4f0b-bf12-6ce4fc238a8f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861043532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861063119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861095093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.86111409Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861131997Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861148384Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861165751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861182708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861200555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861218042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_connection connected from Ok(127.0.0.1:62618)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861249446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_connection connected from Ok(127.0.0.1:49017)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861730793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_connection connected from Ok(127.0.0.1:42984)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.861792282Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862137932Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862171456Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862287496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862308992Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862588144Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862628386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862880713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.862911967Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863308978Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863537548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863563694Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863577761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863590309Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863603817Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863616965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863899915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.86392901Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[0]R flush_numbers[0..12]: [219, 236, 240, 240, 236, 214, 235, 219, 240, 240, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863950207Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[0]R generation[0..12]: [122, 133, 135, 135, 133, 120, 132, 122, 135, 135, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863965754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.863980721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[1]R flush_numbers[0..12]: [219, 236, 240, 240, 236, 214, 235, 219, 240, 240, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864011346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[1]R generation[0..12]: [122, 133, 135, 135, 133, 120, 132, 122, 135, 135, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864026783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864041191Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[2]R flush_numbers[0..12]: [219, 236, 240, 240, 236, 214, 235, 219, 240, 240, 231, 231]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864054938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[2]R generation[0..12]: [122, 133, 135, 135, 133, 120, 132, 122, 135, 135, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864068636Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864082624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864109449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864123326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864136254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:15:10.864149232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"":"downstairs","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864180036Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864208722Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"0","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864235357Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"1","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864260533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"client":"2","":"downstairs","session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:15:10.864285138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864313083Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"6d499d22-6278-4f0b-bf12-6ce4fc238a8f is now active with session: ee1569bc-cca9-4b4d-b6ee-0550357cb845","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864338019Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864356316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726,"session_id":"ee1569bc-cca9-4b4d-b6ee-0550357cb845"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:10.864368924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 28672 data: 12 13 12 12 11 12 13 02/30 Write block 267 len 12288 data: 7 6 7 03/30 Write block 156 len 36864 data: 8 9 10 10 9 9 12 14 15 04/30 Read block 530 len 8192 05/30 Write block 478 len 32768 data: 11 10 12 10 9 12 13 13 06/30 Read block 357 len 24576 07/30 Read block 26 len 4096 08/30 Read block 129 len 36864 09/30 Write block 409 len 40960 data: 11 11 10 11 11 11 13 11 12 9 10/30 Write block 417 len 20480 data: 13 10 10 10 10 11/30 Read block 47 len 40960 12/30 Read block 312 len 4096 13/30 Read block 466 len 20480 14/30 Read block 335 len 40960 15/30 Write block 328 len 36864 data: 11 11 10 13 14 14 16 19 18 16/30 Read block 19 len 20480 17/30 Read block 146 len 40960 18/30 Read block 570 len 12288 19/30 Write block 40 len 16384 data: 9 12 10 11 20/30 Write block 57 len 12288 data: 11 11 8 21/30 Write block 447 len 4096 data: 14 22/30 Read block 532 len 40960 23/30 Read block 522 len 28672 24/30 Read block 114 len 28672 25/30 Write block 219 len 16384 data: 13 9 8 8 26/30 Read block 402 len 20480 27/30 Write block 502 len 28672 data: 11 9 9 9 10 11 11 28/30 Write block 367 len 16384 data: 18 14 13 14 29/30 Read block 392 len 32768 30/30 Read block 349 len 16384 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 7 Done Done Done false 2 Acked 1001 Write 3 Done Done Done false 3 Acked 1002 Write 9 Done Done Done false 4 Acked 1003 Read 2 Done Done Done false 5 Acked 1004 Write 8 Done Done Done false 6 Acked 1005 Read 6 Done Done Done false 7 Acked 1006 Read 1 Done Done Done false 8 Acked 1007 Read 9 Done Done Done false 9 Acked 1008 Write 10 Done Done Done false 10 Acked 1009 Write 5 Done Done Done false 11 Acked 1010 Read 10 Done Done Done false 12 Acked 1011 Read 1 Done Done Done false 13 Acked 1012 Read 5 Done Done Done false 14 Acked 1013 Read 10 Done Done Done false 15 Acked 1014 Write 9 Done Done Done false 16 Acked 1015 Read 5 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:10.884861708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 135 135 135 240 240 240 T T T 3 060-079 135 135 135 240 240 240 F F F 4 080-099 133 133 133 236 236 236 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 132 132 132 235 235 235 F F F 7 140-159 122 122 122 219 219 219 T T T 8 160-179 135 135 135 240 240 240 T T T 9 180-199 135 135 135 240 240 240 F F F 10 200-219 130 130 130 231 231 231 T T T 11 220-239 130 130 130 231 231 231 T T T 12 240-259 135 135 135 240 240 240 T T T 13 260-279 134 134 134 237 237 237 T T T 14 280-299 135 135 135 240 240 240 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 128 128 128 227 227 227 T T T 17 340-359 130 130 130 230 230 230 F F F 18 360-379 135 135 135 240 240 240 T T T 19 380-399 130 130 130 231 231 231 F F F 20 400-419 130 130 130 231 231 231 T T T 21 420-439 129 129 129 229 229 229 T T T 22 440-459 133 133 133 236 236 236 T T T 23 460-479 134 134 134 239 239 239 T T T 24 480-499 133 133 133 236 236 236 T T T 25 500-519 131 131 131 234 234 234 T T T 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 135 135 135 240 240 240 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 135, Max flush: 240 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 136 136 136 241 241 241 F F F 3 060-079 135 135 135 240 240 240 F F F 4 080-099 133 133 133 236 236 236 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 132 132 132 235 235 235 F F F 7 140-159 136 136 136 241 241 241 F F F 8 160-179 136 136 136 241 241 241 F F F 9 180-199 135 135 135 240 240 240 F F F 10 200-219 136 136 136 241 241 241 F F F 11 220-239 136 136 136 241 241 241 F F F 12 240-259 136 136 136 241 241 241 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 135 135 135 240 240 240 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 136 136 136 241 241 241 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 136 136 136 241 241 241 F F F 19 380-399 130 130 130 231 231 231 F F F 20 400-419 136 136 136 241 241 241 F F F 21 420-439 136 136 136 241 241 241 F F F 22 440-459 136 136 136 241 241 241 F F F 23 460-479 136 136 136 241 241 241 F F F 24 480-499 136 136 136 241 241 241 F F F 25 500-519 136 136 136 241 241 241 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 123 123 123 220 220 220 F F F 28 560-579 135 135 135 240 240 240 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 136, Max flush: 241 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.349897072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.394929686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.39582543Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.395863713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.395878861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.395893368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734} {"msg":"Crucible 411df4dc-fa7a-4a4d-8faa-6031906c7534 has session id: a2b51405-1391-4c8c-b884-b59b9e804141","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.396385682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Upstairs opts: Upstairs UUID: 411df4dc-fa7a-4a4d-8faa-6031906c7534, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.396431754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Crucible stats registered with UUID: 411df4dc-fa7a-4a4d-8faa-6031906c7534","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.39645411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427124093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734} {"msg":"411df4dc-fa7a-4a4d-8faa-6031906c7534 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427173315Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427192201Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427221446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427239233Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427348944Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427384418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427409863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.42743249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427457815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427480951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_connection connected from Ok(127.0.0.1:50582)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427502767Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_connection connected from Ok(127.0.0.1:49424)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427527453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_connection connected from Ok(127.0.0.1:49671)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.427552029Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.428345871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.428374656Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.428389943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.428405001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.428468879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.428506513Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429174356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429203611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429226187Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429913408Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429942433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429964069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429978067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.429995364Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.43001749Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430034107Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430047875Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[0]R flush_numbers[0..12]: [219, 236, 241, 240, 236, 214, 235, 241, 241, 240, 241, 241]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430061752Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[0]R generation[0..12]: [122, 133, 136, 135, 133, 120, 132, 136, 136, 135, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.4300758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430089867Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[1]R flush_numbers[0..12]: [219, 236, 241, 240, 236, 214, 235, 241, 241, 240, 241, 241]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430103425Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[1]R generation[0..12]: [122, 133, 136, 135, 133, 120, 132, 136, 136, 135, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430117832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430142988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[2]R flush_numbers[0..12]: [219, 236, 241, 240, 236, 214, 235, 241, 241, 240, 241, 241]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430170863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[2]R generation[0..12]: [122, 133, 136, 135, 133, 120, 132, 136, 136, 135, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430193619Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:15:15.430218025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Max found gen is 137","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430253109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Generation requested: 138 >= found:137","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430272915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430286563Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430310339Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"":"downstairs","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430336174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.43035898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"0","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430385286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"1","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430413641Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"client":"2","":"downstairs","session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430437247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430457383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"411df4dc-fa7a-4a4d-8faa-6031906c7534 is now active with session: a2b51405-1391-4c8c-b884-b59b9e804141","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.43047741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430497496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734,"session_id":"a2b51405-1391-4c8c-b884-b59b9e804141"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:15.430518713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 32768 data: 12 13 15 16 12 10 10 10 02/30 Write block 484 len 32768 data: 14 14 11 11 9 10 7 8 03/30 Flush 04/30 Write block 360 len 36864 data: 11 15 18 17 18 16 18 19 15 05/30 Write block 322 len 4096 data: 4 06/30 Read block 494 len 12288 07/30 Read block 167 len 24576 08/30 Read block 300 len 28672 09/30 Flush 10/30 Write block 408 len 40960 data: 14 12 12 11 12 12 12 14 12 14 11/30 Read block 531 len 28672 12/30 Flush 13/30 Write block 553 len 28672 data: 15 13 11 14 14 11 11 14/30 Write block 515 len 4096 data: 15 15/30 Read block 102 len 24576 16/30 Read block 2 len 24576 17/30 Write block 184 len 12288 data: 12 12 12 18/30 Read block 585 len 20480 19/30 Read block 16 len 12288 20/30 Flush 21/30 Write block 139 len 12288 data: 11 9 7 22/30 Write block 482 len 40960 data: 10 13 15 15 12 12 10 11 8 9 23/30 Read block 35 len 40960 24/30 Write block 207 len 24576 data: 10 10 13 12 10 7 25/30 Read block 132 len 40960 26/30 Read block 247 len 20480 27/30 Read block 257 len 28672 28/30 Write block 396 len 28672 data: 7 9 10 10 11 11 13 29/30 Read block 269 len 24576 30/30 Read block 551 len 40960 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:15.458201395Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 136 136 136 241 241 241 F F F 3 060-079 135 135 135 240 240 240 F F F 4 080-099 133 133 133 236 236 236 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 132 132 132 235 235 235 T T T 7 140-159 136 136 136 241 241 241 T T T 8 160-179 136 136 136 241 241 241 F F F 9 180-199 138 138 138 245 245 245 F F F 10 200-219 136 136 136 241 241 241 T T T 11 220-239 136 136 136 241 241 241 F F F 12 240-259 136 136 136 241 241 241 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 135 135 135 240 240 240 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 138 138 138 243 243 243 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 138 138 138 243 243 243 F F F 19 380-399 130 130 130 231 231 231 T T T 20 400-419 138 138 138 244 244 244 T T T 21 420-439 136 136 136 241 241 241 F F F 22 440-459 136 136 136 241 241 241 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 138 138 138 242 242 242 T T T 25 500-519 138 138 138 245 245 245 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 138 138 138 245 245 245 F F F 28 560-579 135 135 135 240 240 240 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 138, Max flush: 245 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 136 136 136 241 241 241 F F F 3 060-079 135 135 135 240 240 240 F F F 4 080-099 133 133 133 236 236 236 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 139 139 139 246 246 246 F F F 7 140-159 139 139 139 246 246 246 F F F 8 160-179 136 136 136 241 241 241 F F F 9 180-199 138 138 138 245 245 245 F F F 10 200-219 139 139 139 246 246 246 F F F 11 220-239 136 136 136 241 241 241 F F F 12 240-259 136 136 136 241 241 241 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 135 135 135 240 240 240 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 138 138 138 243 243 243 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 138 138 138 243 243 243 F F F 19 380-399 139 139 139 246 246 246 F F F 20 400-419 139 139 139 246 246 246 F F F 21 420-439 136 136 136 241 241 241 F F F 22 440-459 136 136 136 241 241 241 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 139 139 139 246 246 246 F F F 25 500-519 138 138 138 245 245 245 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 138 138 138 245 245 245 F F F 28 560-579 135 135 135 240 240 240 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 139, Max flush: 246 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.793863812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.838479608Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.839456578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.839493462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.83950774Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.839522667Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742} {"msg":"Crucible fc6c5977-76e9-4bd7-88d0-8238546e4c22 has session id: 588faa4c-9192-45e3-8c77-018826fca757","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.840113154Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Upstairs opts: Upstairs UUID: fc6c5977-76e9-4bd7-88d0-8238546e4c22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.840164105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Crucible stats registered with UUID: fc6c5977-76e9-4bd7-88d0-8238546e4c22","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.840184072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871410018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742} {"msg":"fc6c5977-76e9-4bd7-88d0-8238546e4c22 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871470507Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871496943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871543475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871579468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871615962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871650366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871682201Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871716815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871749929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871781133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_connection connected from Ok(127.0.0.1:39074)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871813348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_connection connected from Ok(127.0.0.1:59209)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871847732Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_connection connected from Ok(127.0.0.1:35135)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.871878167Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.872752474Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.872793577Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.872838729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.872882152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.872958788Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.872993962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.873463281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.873506513Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.873548696Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.87415428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874195203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874224238Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874250424Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.87427455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874296646Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874316812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874340328Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[0]R flush_numbers[0..12]: [219, 236, 241, 240, 236, 214, 246, 246, 241, 245, 246, 241]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874365734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[0]R generation[0..12]: [122, 133, 136, 135, 133, 120, 139, 139, 136, 138, 139, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874409036Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874434802Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[1]R flush_numbers[0..12]: [219, 236, 241, 240, 236, 214, 246, 246, 241, 245, 246, 241]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874459377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[1]R generation[0..12]: [122, 133, 136, 135, 133, 120, 139, 139, 136, 138, 139, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874484293Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874510868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[2]R flush_numbers[0..12]: [219, 236, 241, 240, 236, 214, 246, 246, 241, 245, 246, 241]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874535984Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[2]R generation[0..12]: [122, 133, 136, 135, 133, 120, 139, 139, 136, 138, 139, 136]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.8745603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:15:19.874588995Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.87461921Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874646955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.87467315Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874699145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"":"downstairs","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874726111Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874751247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"0","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-09-27T17:15:19.874782031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"1","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874815745Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"client":"2","":"downstairs","session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874841841Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874855648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"fc6c5977-76e9-4bd7-88d0-8238546e4c22 is now active with session: 588faa4c-9192-45e3-8c77-018826fca757","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874868726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874880814Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742,"session_id":"588faa4c-9192-45e3-8c77-018826fca757"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:19.874892952Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 24576 data: 10 12 12 12 12 12 02/30 Write block 540 len 20480 data: 13 13 11 10 10 03/30 Read block 100 len 16384 04/30 Flush 05/30 Write block 214 len 28672 data: 9 7 9 9 10 14 10 06/30 Read block 114 len 28672 07/30 Write block 395 len 12288 data: 6 8 10 08/30 Read block 408 len 20480 09/30 Write block 459 len 4096 data: 11 10/30 Write block 498 len 8192 data: 9 9 11/30 Read block 13 len 12288 12/30 Write block 566 len 12288 data: 7 9 8 13/30 Write block 426 len 12288 data: 6 7 8 14/30 Write block 572 len 28672 data: 12 12 12 13 12 11 10 15/30 Flush 16/30 Write block 501 len 28672 data: 10 12 10 10 10 11 12 17/30 Read block 304 len 24576 18/30 Flush 19/30 Write block 396 len 8192 data: 9 11 20/30 Flush 21/30 Write block 405 len 24576 data: 14 15 15 15 13 13 22/30 Flush 23/30 Write block 517 len 36864 data: 15 14 11 11 12 13 11 12 10 24/30 Read block 434 len 8192 25/30 Read block 474 len 20480 26/30 Read block 469 len 4096 27/30 Write block 132 len 8192 data: 9 7 28/30 Read block 411 len 32768 29/30 Write block 92 len 4096 data: 6 30/30 Read block 148 len 28672 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 1 Done Sent Done false 30 Acked 1029 Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 7 6 8 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:19.903562563Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 136 136 136 241 241 241 F F F 3 060-079 135 135 135 240 240 240 F F F 4 080-099 133 133 133 236 236 236 T T T 5 100-119 120 120 120 214 214 214 F F F 6 120-139 139 139 139 246 246 246 T T T 7 140-159 139 139 139 246 246 246 F F F 8 160-179 136 136 136 241 241 241 F F F 9 180-199 140 140 140 247 247 247 F F F 10 200-219 140 140 140 248 248 248 F F F 11 220-239 140 140 140 248 248 248 F F F 12 240-259 136 136 136 241 241 241 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 135 135 135 240 240 240 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 138 138 138 243 243 243 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 138 138 138 243 243 243 F F F 19 380-399 140 140 140 250 250 250 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 140 140 140 248 248 248 F F F 25 500-519 140 140 140 249 249 249 T T T 26 520-539 127 127 127 222 222 222 T T T 27 540-559 140 140 140 247 247 247 F F F 28 560-579 140 140 140 248 248 248 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 140, Max flush: 251 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 136 136 136 241 241 241 F F F 3 060-079 135 135 135 240 240 240 F F F 4 080-099 141 141 141 252 252 252 F F F 5 100-119 120 120 120 214 214 214 F F F 6 120-139 141 141 141 252 252 252 F F F 7 140-159 139 139 139 246 246 246 F F F 8 160-179 136 136 136 241 241 241 F F F 9 180-199 140 140 140 247 247 247 F F F 10 200-219 140 140 140 248 248 248 F F F 11 220-239 140 140 140 248 248 248 F F F 12 240-259 136 136 136 241 241 241 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 135 135 135 240 240 240 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 138 138 138 243 243 243 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 138 138 138 243 243 243 F F F 19 380-399 140 140 140 250 250 250 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 140 140 140 248 248 248 F F F 25 500-519 141 141 141 252 252 252 F F F 26 520-539 141 141 141 252 252 252 F F F 27 540-559 140 140 140 247 247 247 F F F 28 560-579 140 140 140 248 248 248 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 141, Max flush: 252 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.214991248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.259679951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.260623207Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.260660361Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.260675598Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.260690136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750} {"msg":"Crucible f6d73017-b5ca-4d5f-9a89-8565102a8726 has session id: 93c4fed6-928d-43a7-9567-79af123cad3f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.26118084Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Upstairs opts: Upstairs UUID: f6d73017-b5ca-4d5f-9a89-8565102a8726, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.261215484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Crucible stats registered with UUID: f6d73017-b5ca-4d5f-9a89-8565102a8726","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.261230532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293290023Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750} {"msg":"f6d73017-b5ca-4d5f-9a89-8565102a8726 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293344603Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.29336532Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293405582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293441646Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293484439Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293517993Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293566525Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293606127Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293644501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293682324Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45861)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293720448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38003)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293751182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57852)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.293781597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.294659974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.294701517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.294732862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.294767176Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.294874657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.294906751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.295385968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.29543137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.295612358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296050952Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296078057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296092805Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296105333Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.2961187Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296131838Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296265005Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.2962946Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[0]R flush_numbers[0..12]: [219, 236, 241, 240, 252, 214, 252, 246, 241, 247, 248, 248]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296310597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[0]R generation[0..12]: [122, 133, 136, 135, 141, 120, 141, 139, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296325334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296339562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[1]R flush_numbers[0..12]: [219, 236, 241, 240, 252, 214, 252, 246, 241, 247, 248, 248]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296353549Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[1]R generation[0..12]: [122, 133, 136, 135, 141, 120, 141, 139, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296367347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296385134Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[2]R flush_numbers[0..12]: [219, 236, 241, 240, 252, 214, 252, 246, 241, 247, 248, 248]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296399652Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[2]R generation[0..12]: [122, 133, 136, 135, 141, 120, 141, 139, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296413429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296427377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296440314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296453312Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.29646621Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296479258Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"":"downstairs","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296492295Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296504163Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"0","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Transition from WaitQuorum to Active","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296529369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"1","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296542616Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"client":"2","":"downstairs","session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296555554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296567792Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"f6d73017-b5ca-4d5f-9a89-8565102a8726 is now active with session: 93c4fed6-928d-43a7-9567-79af123cad3f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.29658043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296594347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750,"session_id":"93c4fed6-928d-43a7-9567-79af123cad3f"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:24.296606645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 126 len 20480 data: 10 11 8 7 8 02/30 Read block 117 len 24576 03/30 Write block 257 len 12288 data: 9 8 10 04/30 Flush 05/30 Write block 394 len 8192 data: 6 7 06/30 Flush 07/30 Flush 08/30 Write block 451 len 28672 data: 13 11 13 15 14 17 16 09/30 Read block 390 len 36864 10/30 Read block 242 len 36864 11/30 Read block 489 len 24576 12/30 Write block 157 len 36864 data: 10 11 11 10 10 13 15 16 13 13/30 Write block 76 len 36864 data: 14 9 9 12 12 11 12 13 14 14/30 Flush 15/30 Write block 218 len 12288 data: 11 15 11 16/30 Read block 539 len 40960 17/30 Read block 382 len 16384 18/30 Write block 320 len 28672 data: 6 5 5 4 6 7 7 19/30 Flush 20/30 Write block 574 len 8192 data: 13 14 21/30 Read block 428 len 4096 22/30 Write block 101 len 32768 data: 11 12 13 10 10 12 10 9 23/30 Write block 213 len 28672 data: 8 10 8 10 10 12 16 24/30 Read block 462 len 40960 25/30 Flush 26/30 Write block 290 len 32768 data: 8 8 8 6 7 5 6 8 27/30 Write block 219 len 40960 data: 17 12 9 9 7 8 8 8 8 8 28/30 Read block 367 len 24576 29/30 Read block 13 len 36864 30/30 Read block 241 len 36864 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:24.342742957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 136 136 136 241 241 241 F F F 3 060-079 142 142 142 256 256 256 F F F 4 080-099 142 142 142 256 256 256 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 142 142 142 253 253 253 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 140 140 140 247 247 247 F F F 10 200-219 142 142 142 258 258 258 T T T 11 220-239 142 142 142 257 257 257 T T T 12 240-259 142 142 142 253 253 253 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 135 135 135 240 240 240 T T T 15 300-319 135 135 135 240 240 240 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 138 138 138 243 243 243 F F F 19 380-399 142 142 142 254 254 254 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 140 140 140 248 248 248 F F F 25 500-519 141 141 141 252 252 252 F F F 26 520-539 141 141 141 252 252 252 F F F 27 540-559 140 140 140 247 247 247 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 142, Max flush: 258 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 133 133 133 236 236 236 F F F 2 040-059 136 136 136 241 241 241 F F F 3 060-079 142 142 142 256 256 256 F F F 4 080-099 142 142 142 256 256 256 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 142 142 142 253 253 253 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 140 140 140 247 247 247 F F F 10 200-219 143 143 143 259 259 259 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 142 142 142 253 253 253 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 143 143 143 259 259 259 F F F 15 300-319 135 135 135 240 240 240 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 130 130 130 230 230 230 F F F 18 360-379 138 138 138 243 243 243 F F F 19 380-399 142 142 142 254 254 254 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 140 140 140 248 248 248 F F F 25 500-519 141 141 141 252 252 252 F F F 26 520-539 141 141 141 252 252 252 F F F 27 540-559 140 140 140 247 247 247 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 143, Max flush: 259 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.632924349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.677940045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.678884311Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.678921435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.678940881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.678955729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758} {"msg":"Crucible fc9e9ebb-cbec-4b10-a0c7-a92f24949660 has session id: 827145dc-ed81-469b-9e43-7ee4f83e2711","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.679382974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Upstairs opts: Upstairs UUID: fc9e9ebb-cbec-4b10-a0c7-a92f24949660, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.679429746Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Crucible stats registered with UUID: fc9e9ebb-cbec-4b10-a0c7-a92f24949660","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.679456361Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710559559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758} {"msg":"fc9e9ebb-cbec-4b10-a0c7-a92f24949660 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710604721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710636316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710687907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.71072673Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710758594Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710790569Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710816674Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710848639Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710883233Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710927155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_connection connected from Ok(127.0.0.1:33556)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710969008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_connection connected from Ok(127.0.0.1:47288)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.710998253Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_connection connected from Ok(127.0.0.1:39740)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.711028817Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.711668006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.711710049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.711743813Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.711771438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.712010367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.712041261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.712431523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.712462048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.712747318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713147339Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713176864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.71319548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713208318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713221946Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713235133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713372419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713400954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[0]R flush_numbers[0..12]: [219, 236, 241, 256, 256, 258, 253, 256, 256, 247, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713416762Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[0]R generation[0..12]: [122, 133, 136, 142, 142, 142, 142, 142, 142, 140, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713431159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713445057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[1]R flush_numbers[0..12]: [219, 236, 241, 256, 256, 258, 253, 256, 256, 247, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713458864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[1]R generation[0..12]: [122, 133, 136, 142, 142, 142, 142, 142, 142, 140, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713472632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.71348651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[2]R flush_numbers[0..12]: [219, 236, 241, 256, 256, 258, 253, 256, 256, 247, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713500297Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[2]R generation[0..12]: [122, 133, 136, 142, 142, 142, 142, 142, 142, 140, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713513865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713531272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.71354516Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713558317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713571175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713584203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"":"downstairs","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.71359732Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713621346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"0","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713634784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"1","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713647871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"client":"2","":"downstairs","session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713661159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713673447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"fc9e9ebb-cbec-4b10-a0c7-a92f24949660 is now active with session: 827145dc-ed81-469b-9e43-7ee4f83e2711","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713686525Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713699053Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758,"session_id":"827145dc-ed81-469b-9e43-7ee4f83e2711"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:28.713711351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 36 len 24576 data: 10 9 10 10 10 13 02/30 Write block 118 len 12288 data: 13 13 10 03/30 Write block 197 len 12288 data: 13 13 13 04/30 Write block 111 len 16384 data: 9 10 7 9 05/30 Read block 119 len 36864 06/30 Read block 533 len 28672 07/30 Write block 495 len 32768 data: 6 6 9 10 10 8 11 13 08/30 Read block 274 len 12288 09/30 Read block 290 len 32768 10/30 Write block 302 len 4096 data: 6 11/30 Read block 515 len 32768 12/30 Read block 360 len 16384 13/30 Write block 183 len 8192 data: 12 13 14/30 Write block 385 len 28672 data: 11 12 11 11 11 10 11 15/30 Read block 551 len 8192 16/30 Read block 59 len 28672 17/30 Flush 18/30 Write block 349 len 32768 data: 7 7 5 6 7 6 6 4 19/30 Flush 20/30 Write block 300 len 12288 data: 6 6 7 21/30 Write block 483 len 28672 data: 14 16 16 13 13 11 12 22/30 Read block 54 len 20480 23/30 Read block 560 len 28672 24/30 Write block 42 len 32768 data: 11 12 10 10 9 10 10 9 25/30 Read block 193 len 40960 26/30 Write block 377 len 4096 data: 11 27/30 Read block 512 len 12288 28/30 Write block 287 len 40960 data: 9 10 10 9 9 9 7 8 6 7 29/30 Read block 262 len 12288 30/30 Write block 511 len 24576 data: 10 11 11 13 16 17 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:28.737814126Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 144 144 144 260 260 260 F F F 2 040-059 144 144 144 260 260 260 T T T 3 060-079 142 142 142 256 256 256 F F F 4 080-099 142 142 142 256 256 256 F F F 5 100-119 144 144 144 260 260 260 F F F 6 120-139 144 144 144 260 260 260 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 144 144 144 260 260 260 F F F 10 200-219 143 143 143 259 259 259 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 142 142 142 253 253 253 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 143 143 143 259 259 259 T T T 15 300-319 144 144 144 260 260 260 T T T 16 320-339 142 142 142 257 257 257 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 138 138 138 243 243 243 T T T 19 380-399 144 144 144 260 260 260 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 144 144 144 260 260 260 T T T 25 500-519 144 144 144 260 260 260 T T T 26 520-539 141 141 141 252 252 252 F F F 27 540-559 140 140 140 247 247 247 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 144, Max flush: 261 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 F F F 1 020-039 144 144 144 260 260 260 F F F 2 040-059 145 145 145 262 262 262 F F F 3 060-079 142 142 142 256 256 256 F F F 4 080-099 142 142 142 256 256 256 F F F 5 100-119 144 144 144 260 260 260 F F F 6 120-139 144 144 144 260 260 260 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 144 144 144 260 260 260 F F F 10 200-219 143 143 143 259 259 259 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 142 142 142 253 253 253 F F F 13 260-279 136 136 136 241 241 241 F F F 14 280-299 145 145 145 262 262 262 F F F 15 300-319 145 145 145 262 262 262 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 144 144 144 260 260 260 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 145 145 145 262 262 262 F F F 25 500-519 145 145 145 262 262 262 F F F 26 520-539 141 141 141 252 252 252 F F F 27 540-559 140 140 140 247 247 247 F F F 28 560-579 142 142 142 258 258 258 F F F 29 580-599 135 135 135 240 240 240 F F F Max gen: 145, Max flush: 262 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.072458221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:33.115087213Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.115683829Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.147749559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.148726369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.148763123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.14877759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.148796647Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766} {"msg":"Crucible a939eef3-00df-4aa0-bed4-9a5e95d6c8c5 has session id: 03b916e3-82d5-4547-8db8-72d1a250d7be","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.149241729Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Upstairs opts: Upstairs UUID: a939eef3-00df-4aa0-bed4-9a5e95d6c8c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.149285792Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Crucible stats registered with UUID: a939eef3-00df-4aa0-bed4-9a5e95d6c8c5","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.149303449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.180916387Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766} {"msg":"a939eef3-00df-4aa0-bed4-9a5e95d6c8c5 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.180974147Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.180994154Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181023759Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181041846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181058832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181075689Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181092397Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181118062Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181153526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.18118519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_connection connected from Ok(127.0.0.1:52331)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181233262Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_connection connected from Ok(127.0.0.1:40587)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181266756Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_connection connected from Ok(127.0.0.1:38455)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.181298611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.182217951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.182262573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.182293398Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.182317713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.182538945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.18256768Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183060494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183097678Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183171645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183676447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.18371713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183748004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.18377523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183790857Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183804345Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183816323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.18382945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[0]R flush_numbers[0..12]: [219, 260, 262, 256, 256, 260, 260, 256, 256, 260, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183868034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[0]R generation[0..12]: [122, 144, 145, 142, 142, 144, 144, 142, 142, 144, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183883521Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183897718Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[1]R flush_numbers[0..12]: [219, 260, 262, 256, 256, 260, 260, 256, 256, 260, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183911356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[1]R generation[0..12]: [122, 144, 145, 142, 142, 144, 144, 142, 142, 144, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183924884Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183940351Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[2]R flush_numbers[0..12]: [219, 260, 262, 256, 256, 260, 260, 256, 256, 260, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183954369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[2]R generation[0..12]: [122, 144, 145, 142, 142, 144, 144, 142, 142, 144, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183968076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183982004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.183997191Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184010559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184023607Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184036894Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"":"downstairs","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184050332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.18406222Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"0","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:15:33.184075467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"1","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184098963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"client":"2","":"downstairs","session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184112021Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184124079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"a939eef3-00df-4aa0-bed4-9a5e95d6c8c5 is now active with session: 03b916e3-82d5-4547-8db8-72d1a250d7be","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184136877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184148655Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766,"session_id":"03b916e3-82d5-4547-8db8-72d1a250d7be"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:33.184160653Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 587 len 24576 data: 6 6 8 11 12 14 02/30 Read block 12 len 16384 03/30 Write block 545 len 8192 data: 9 12 04/30 Write block 274 len 32768 data: 9 9 11 10 9 7 8 8 05/30 Write block 193 len 36864 data: 8 9 11 13 14 14 14 13 13 06/30 Write block 337 len 16384 data: 18 20 16 16 07/30 Read block 486 len 40960 08/30 Write block 575 len 24576 data: 15 13 12 11 13 12 09/30 Flush 10/30 Write block 79 len 24576 data: 13 13 12 13 14 15 11/30 Read block 90 len 20480 12/30 Flush 13/30 Write block 344 len 16384 data: 12 7 8 6 14/30 Read block 179 len 16384 15/30 Read block 492 len 36864 16/30 Read block 484 len 8192 17/30 Write block 181 len 12288 data: 10 12 13 18/30 Flush 19/30 Write block 384 len 32768 data: 11 12 13 12 12 12 11 12 20/30 Read block 78 len 12288 21/30 Write block 503 len 24576 data: 11 11 11 12 13 12 22/30 Read block 204 len 24576 23/30 Write block 359 len 4096 data: 11 24/30 Read block 286 len 36864 25/30 Read block 501 len 8192 26/30 Write block 280 len 40960 data: 9 9 10 10 8 8 9 10 11 11 27/30 Write block 124 len 16384 data: 15 12 11 12 28/30 Write block 7 len 12288 data: 14 11 10 29/30 Read block 100 len 12288 30/30 Read block 444 len 32768 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Read 3 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 4 Done Sent Sent false 28 Acked 1027 Write 3 Done Sent Sent false 29 Acked 1028 Read 3 Done Sent Sent false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 4 4 8 Done 12 8 8 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:33.210456817Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 219 219 219 T T T 1 020-039 144 144 144 260 260 260 F F F 2 040-059 145 145 145 262 262 262 F F F 3 060-079 146 146 146 264 264 264 F F F 4 080-099 146 146 146 264 264 264 F F F 5 100-119 144 144 144 260 260 260 F F F 6 120-139 144 144 144 260 260 260 T T T 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 146 146 146 265 265 265 F F F 10 200-219 146 146 146 263 263 263 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 142 142 142 253 253 253 F F F 13 260-279 146 146 146 263 263 263 F F F 14 280-299 146 146 146 263 263 263 T T T 15 300-319 145 145 145 262 262 262 F F F 16 320-339 146 146 146 263 263 263 F F F 17 340-359 146 146 146 265 265 265 T T T 18 360-379 145 145 145 262 262 262 F F F 19 380-399 144 144 144 260 260 260 T T T 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 145 145 145 262 262 262 F F F 25 500-519 145 145 145 262 262 262 T T T 26 520-539 141 141 141 252 252 252 F F F 27 540-559 146 146 146 263 263 263 F F F 28 560-579 146 146 146 263 263 263 F F F 29 580-599 146 146 146 263 263 263 F F F Max gen: 146, Max flush: 265 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 144 144 144 260 260 260 F F F 2 040-059 145 145 145 262 262 262 F F F 3 060-079 146 146 146 264 264 264 F F F 4 080-099 146 146 146 264 264 264 F F F 5 100-119 144 144 144 260 260 260 F F F 6 120-139 147 147 147 266 266 266 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 146 146 146 265 265 265 F F F 10 200-219 146 146 146 263 263 263 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 142 142 142 253 253 253 F F F 13 260-279 146 146 146 263 263 263 F F F 14 280-299 147 147 147 266 266 266 F F F 15 300-319 145 145 145 262 262 262 F F F 16 320-339 146 146 146 263 263 263 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 147 147 147 266 266 266 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 242 242 242 F F F 24 480-499 145 145 145 262 262 262 F F F 25 500-519 147 147 147 266 266 266 F F F 26 520-539 141 141 141 252 252 252 F F F 27 540-559 146 146 146 263 263 263 F F F 28 560-579 146 146 146 263 263 263 F F F 29 580-599 146 146 146 263 263 263 F F F Max gen: 147, Max flush: 266 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.590847921Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.636866833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.637839434Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.637889145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.637907232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.637925449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774} {"msg":"Crucible b52f6090-5d4e-4a29-b91e-e96ae0c4db41 has session id: 9be1e233-c401-4da7-9360-816f677b901a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.638396737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Upstairs opts: Upstairs UUID: b52f6090-5d4e-4a29-b91e-e96ae0c4db41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.638458186Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Crucible stats registered with UUID: b52f6090-5d4e-4a29-b91e-e96ae0c4db41","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.638488751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670253173Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774} {"msg":"b52f6090-5d4e-4a29-b91e-e96ae0c4db41 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670310173Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.67032971Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670360324Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670378761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670396428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670413205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670429992Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670453618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670483773Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670512728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37838)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.67061752Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63807)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670659812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41983)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.670694376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.671739404Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.671769179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.671784627Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.671799074Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.671812862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.671826009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.672420106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.672475806Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.672613962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.672967111Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673005944Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673035169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673140801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673169016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673184283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673206589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673220927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[0]R flush_numbers[0..12]: [266, 260, 262, 264, 264, 260, 266, 256, 256, 265, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673292374Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[0]R generation[0..12]: [147, 144, 145, 146, 146, 144, 147, 142, 142, 146, 146, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673312741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673327178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[1]R flush_numbers[0..12]: [266, 260, 262, 264, 264, 260, 266, 256, 256, 265, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673340816Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[1]R generation[0..12]: [147, 144, 145, 146, 146, 144, 147, 142, 142, 146, 146, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673354823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673369151Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[2]R flush_numbers[0..12]: [266, 260, 262, 264, 264, 260, 266, 256, 256, 265, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673383078Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[2]R generation[0..12]: [147, 144, 145, 146, 146, 144, 147, 142, 142, 146, 146, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673396676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673410554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673423431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673436379Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673449207Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673464694Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"":"downstairs","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:15:37.673477902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673497838Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"0","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673511066Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"1","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673524084Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"client":"2","":"downstairs","session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673537112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673549329Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"b52f6090-5d4e-4a29-b91e-e96ae0c4db41 is now active with session: 9be1e233-c401-4da7-9360-816f677b901a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673561947Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673573865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774,"session_id":"9be1e233-c401-4da7-9360-816f677b901a"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:37.673586003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 12288 data: 14 14 12 02/30 Read block 565 len 40960 03/30 Read block 326 len 8192 04/30 Read block 45 len 40960 05/30 Write block 573 len 4096 data: 13 06/30 Write block 245 len 28672 data: 11 10 7 10 9 9 9 07/30 Write block 591 len 20480 data: 13 15 13 12 9 08/30 Read block 92 len 20480 09/30 Write block 40 len 28672 data: 11 14 12 13 11 11 10 10/30 Read block 250 len 36864 11/30 Write block 182 len 28672 data: 13 14 14 13 13 11 10 12/30 Read block 136 len 24576 13/30 Write block 109 len 32768 data: 9 10 10 11 8 10 6 9 14/30 Write block 80 len 24576 data: 14 13 14 15 16 13 15/30 Flush 16/30 Flush 17/30 Write block 102 len 8192 data: 13 14 18/30 Read block 264 len 16384 19/30 Read block 83 len 16384 20/30 Read block 107 len 40960 21/30 Flush 22/30 Write block 34 len 4096 data: 13 23/30 Write block 246 len 16384 data: 11 8 11 10 24/30 Read block 583 len 24576 25/30 Write block 464 len 12288 data: 17 13 11 26/30 Write block 367 len 36864 data: 20 16 14 15 16 16 14 14 15 27/30 Write block 452 len 28672 data: 12 14 16 15 18 17 14 28/30 Write block 447 len 24576 data: 15 12 12 14 14 13 29/30 Write block 293 len 24576 data: 8 9 7 8 9 7 30/30 Write block 379 len 12288 data: 10 8 11 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:37.698145179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 144 144 144 260 260 260 T T T 2 040-059 148 148 148 267 267 267 F F F 3 060-079 146 146 146 264 264 264 F F F 4 080-099 148 148 148 267 267 267 F F F 5 100-119 148 148 148 269 269 269 F F F 6 120-139 147 147 147 266 266 266 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 148 148 148 267 267 267 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 148 148 148 267 267 267 T T T 13 260-279 146 146 146 263 263 263 F F F 14 280-299 147 147 147 266 266 266 T T T 15 300-319 145 145 145 262 262 262 F F F 16 320-339 146 146 146 263 263 263 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 145 145 145 262 262 262 T T T 19 380-399 147 147 147 266 266 266 T T T 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 142 142 142 256 256 256 T T T 23 460-479 138 138 138 242 242 242 T T T 24 480-499 145 145 145 262 262 262 F F F 25 500-519 147 147 147 266 266 266 F F F 26 520-539 141 141 141 252 252 252 F F F 27 540-559 146 146 146 263 263 263 F F F 28 560-579 148 148 148 267 267 267 F F F 29 580-599 148 148 148 267 267 267 F F F Max gen: 148, Max flush: 269 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 149 149 149 270 270 270 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 146 146 146 264 264 264 F F F 4 080-099 148 148 148 267 267 267 F F F 5 100-119 148 148 148 269 269 269 F F F 6 120-139 147 147 147 266 266 266 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 148 148 148 267 267 267 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 146 146 146 263 263 263 F F F 14 280-299 149 149 149 270 270 270 F F F 15 300-319 145 145 145 262 262 262 F F F 16 320-339 146 146 146 263 263 263 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 149 149 149 270 270 270 F F F 19 380-399 149 149 149 270 270 270 F F F 20 400-419 140 140 140 251 251 251 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 149 149 149 270 270 270 F F F 23 460-479 149 149 149 270 270 270 F F F 24 480-499 145 145 145 262 262 262 F F F 25 500-519 147 147 147 266 266 266 F F F 26 520-539 141 141 141 252 252 252 F F F 27 540-559 146 146 146 263 263 263 F F F 28 560-579 148 148 148 267 267 267 F F F 29 580-599 148 148 148 267 267 267 F F F Max gen: 149, Max flush: 270 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.050700017Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.095170289Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.096118614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.096156607Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.096171435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.096186132Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782} {"msg":"Crucible d90c270c-0b0c-43d8-af14-db9fa979a81c has session id: 6160ddfd-9e2d-4811-97bc-e53dd607c11f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.096723908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Upstairs opts: Upstairs UUID: d90c270c-0b0c-43d8-af14-db9fa979a81c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.096766491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Crucible stats registered with UUID: d90c270c-0b0c-43d8-af14-db9fa979a81c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.096790777Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.127846403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782} {"msg":"d90c270c-0b0c-43d8-af14-db9fa979a81c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.127910531Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.127933667Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.127967821Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.127989138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128007904Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128024972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128041558Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128057936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128075463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128103548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63112)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128558728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34547)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128620078Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52118)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.128659891Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.129246919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.129276294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.129346082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.129377756Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.129394703Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.129408701Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130001708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130031482Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.13004772Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130657643Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130696996Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130725342Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130746918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130769984Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.13079248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130814326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130840282Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[0]R flush_numbers[0..12]: [266, 270, 267, 264, 267, 269, 266, 256, 256, 267, 267, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130864517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[0]R generation[0..12]: [147, 149, 148, 146, 148, 148, 147, 142, 142, 148, 148, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130892302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130917298Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[1]R flush_numbers[0..12]: [266, 270, 267, 264, 267, 269, 266, 256, 256, 267, 267, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130943034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[1]R generation[0..12]: [147, 149, 148, 146, 148, 148, 147, 142, 142, 148, 148, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130969579Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.130994645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[2]R flush_numbers[0..12]: [266, 270, 267, 264, 267, 269, 266, 256, 256, 267, 267, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.13102003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"[2]R generation[0..12]: [147, 149, 148, 146, 148, 148, 147, 142, 142, 148, 148, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131048385Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131086869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131115384Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131142599Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131169114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.13119597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"":"downstairs","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131222755Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"0","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131282705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"1","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131311869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"client":"2","":"downstairs","session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131336735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131350233Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"d90c270c-0b0c-43d8-af14-db9fa979a81c is now active with session: 6160ddfd-9e2d-4811-97bc-e53dd607c11f","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.13136338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131375468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782,"session_id":"6160ddfd-9e2d-4811-97bc-e53dd607c11f"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:42.131388546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 123 len 12288 data: 13 16 13 03/30 Read block 370 len 20480 04/30 Read block 398 len 24576 05/30 Flush 06/30 Flush 07/30 Write block 534 len 24576 data: 10 12 10 11 11 12 08/30 Write block 182 len 16384 data: 14 15 15 14 09/30 Write block 501 len 24576 data: 12 14 12 12 12 13 10/30 Write block 261 len 28672 data: 13 12 11 9 7 8 8 11/30 Write block 413 len 16384 data: 13 13 15 13 12/30 Write block 171 len 4096 data: 13 13/30 Read block 107 len 20480 14/30 Read block 260 len 32768 15/30 Write block 565 len 8192 data: 9 8 16/30 Write block 332 len 20480 data: 15 15 17 20 19 17/30 Read block 141 len 40960 18/30 Write block 173 len 24576 data: 10 11 9 9 9 9 19/30 Read block 70 len 4096 20/30 Write block 562 len 16384 data: 11 9 9 10 21/30 Read block 420 len 8192 22/30 Flush 23/30 Write block 277 len 40960 data: 11 10 8 10 10 11 11 9 9 10 24/30 Write block 127 len 8192 data: 13 9 25/30 Write block 105 len 8192 data: 11 13 26/30 Read block 384 len 24576 27/30 Read block 513 len 8192 28/30 Read block 68 len 24576 29/30 Read block 537 len 36864 30/30 Write block 497 len 20480 data: 10 11 11 9 13 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:42.15417961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 149 149 149 270 270 270 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 146 146 146 264 264 264 F F F 4 080-099 148 148 148 267 267 267 F F F 5 100-119 148 148 148 269 269 269 T T T 6 120-139 150 150 150 272 272 272 T T T 7 140-159 142 142 142 256 256 256 F F F 8 160-179 150 150 150 274 274 274 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 150 150 150 274 274 274 T T T 14 280-299 149 149 149 270 270 270 T T T 15 300-319 145 145 145 262 262 262 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 149 149 149 270 270 270 F F F 19 380-399 149 149 149 270 270 270 F F F 20 400-419 150 150 150 274 274 274 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 149 149 149 270 270 270 F F F 23 460-479 149 149 149 270 270 270 F F F 24 480-499 145 145 145 262 262 262 T T T 25 500-519 150 150 150 274 274 274 T T T 26 520-539 150 150 150 274 274 274 F F F 27 540-559 146 146 146 263 263 263 F F F 28 560-579 150 150 150 274 274 274 F F F 29 580-599 148 148 148 267 267 267 F F F Max gen: 150, Max flush: 274 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 149 149 149 270 270 270 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 146 146 146 264 264 264 F F F 4 080-099 148 148 148 267 267 267 F F F 5 100-119 151 151 151 275 275 275 F F F 6 120-139 151 151 151 275 275 275 F F F 7 140-159 142 142 142 256 256 256 F F F 8 160-179 150 150 150 274 274 274 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 143 143 143 259 259 259 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 151 151 151 275 275 275 F F F 14 280-299 151 151 151 275 275 275 F F F 15 300-319 145 145 145 262 262 262 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 149 149 149 270 270 270 F F F 19 380-399 149 149 149 270 270 270 F F F 20 400-419 150 150 150 274 274 274 F F F 21 420-439 140 140 140 248 248 248 F F F 22 440-459 149 149 149 270 270 270 F F F 23 460-479 149 149 149 270 270 270 F F F 24 480-499 151 151 151 275 275 275 F F F 25 500-519 151 151 151 275 275 275 F F F 26 520-539 150 150 150 274 274 274 F F F 27 540-559 146 146 146 263 263 263 F F F 28 560-579 150 150 150 274 274 274 F F F 29 580-599 148 148 148 267 267 267 F F F Max gen: 151, Max flush: 275 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.502547467Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.547230782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.548154181Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.548190774Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.548205102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.548219389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790} {"msg":"Crucible d0ee504e-3307-46f8-b6b7-dc1534cc80a6 has session id: be82dd0d-4a20-41c6-bc6e-42acd2dde746","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.548653534Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Upstairs opts: Upstairs UUID: d0ee504e-3307-46f8-b6b7-dc1534cc80a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.548796329Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Crucible stats registered with UUID: d0ee504e-3307-46f8-b6b7-dc1534cc80a6","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.548823354Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579660598Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790} {"msg":"d0ee504e-3307-46f8-b6b7-dc1534cc80a6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579734275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579766449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579812042Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579850175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579889678Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579927292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.579957326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.580019465Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.580137085Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.580174369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_connection connected from Ok(127.0.0.1:57079)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.580198744Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_connection connected from Ok(127.0.0.1:64657)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.58027862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_connection connected from Ok(127.0.0.1:48614)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.580320043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.580994496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581037259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581070523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581099138Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581129862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581154968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581781279Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581822582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.581854736Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582475388Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582514242Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582543216Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582567572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582591068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582612914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582633631Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582657666Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[0]R flush_numbers[0..12]: [266, 270, 267, 264, 267, 275, 275, 256, 274, 274, 267, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582683092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[0]R generation[0..12]: [147, 149, 148, 146, 148, 151, 151, 142, 150, 150, 148, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582706538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582729794Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[1]R flush_numbers[0..12]: [266, 270, 267, 264, 267, 275, 275, 256, 274, 274, 267, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.58275349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[1]R generation[0..12]: [147, 149, 148, 146, 148, 151, 151, 142, 150, 150, 148, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582782325Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.58281133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[2]R flush_numbers[0..12]: [266, 270, 267, 264, 267, 275, 275, 256, 274, 274, 267, 259]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582835326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[2]R generation[0..12]: [147, 149, 148, 146, 148, 151, 151, 142, 150, 150, 148, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582860541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582889706Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582928979Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.582957324Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.58298426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583010765Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"":"downstairs","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.58303892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583063606Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"0","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:15:46.583091161Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"1","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583120506Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"client":"2","":"downstairs","session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583147681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583173137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"d0ee504e-3307-46f8-b6b7-dc1534cc80a6 is now active with session: be82dd0d-4a20-41c6-bc6e-42acd2dde746","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583191433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583203941Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790,"session_id":"be82dd0d-4a20-41c6-bc6e-42acd2dde746"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:46.583216169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 21 len 20480 data: 9 7 7 8 8 02/30 Read block 207 len 32768 03/30 Write block 587 len 12288 data: 7 7 9 04/30 Write block 226 len 4096 data: 9 05/30 Write block 394 len 8192 data: 7 8 06/30 Read block 154 len 12288 07/30 Write block 71 len 24576 data: 13 13 9 13 13 15 08/30 Write block 419 len 32768 data: 11 11 11 10 8 9 10 7 09/30 Write block 583 len 16384 data: 13 9 10 7 10/30 Read block 319 len 28672 11/30 Flush 12/30 Write block 149 len 24576 data: 7 8 9 9 8 8 13/30 Read block 512 len 24576 14/30 Read block 164 len 8192 15/30 Read block 189 len 40960 16/30 Write block 264 len 32768 data: 10 8 9 9 7 8 7 8 17/30 Write block 361 len 16384 data: 16 19 18 19 18/30 Read block 302 len 36864 19/30 Read block 248 len 32768 20/30 Read block 427 len 16384 21/30 Read block 209 len 28672 22/30 Write block 394 len 4096 data: 8 23/30 Read block 235 len 28672 24/30 Write block 562 len 28672 data: 12 10 10 11 9 10 9 25/30 Write block 533 len 40960 data: 11 11 13 11 12 12 13 14 14 12 26/30 Write block 307 len 36864 data: 13 14 13 15 15 13 8 7 7 27/30 Read block 52 len 20480 28/30 Write block 542 len 16384 data: 13 11 11 10 29/30 Write block 94 len 32768 data: 10 10 12 12 12 13 12 12 30/30 Write block 452 len 12288 data: 14 15 17 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 12 Acked 1011 Write 6 Done Done Done false 13 Acked 1012 Read 6 Done Done Done false 14 Acked 1013 Read 2 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Write 8 Done Done Done false 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Write 10 Sent Done Sent false 26 Acked 1025 Write 9 Sent Done Sent false 27 Acked 1026 Read 5 Sent Done Sent false 28 Acked 1027 Write 4 Sent Sent Sent false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 6 3 6 15 Done 13 16 13 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:46.604199918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 152 152 152 276 276 276 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 148 148 148 267 267 267 T T T 5 100-119 151 151 151 275 275 275 T T T 6 120-139 151 151 151 275 275 275 F F F 7 140-159 142 142 142 256 256 256 T T T 8 160-179 150 150 150 274 274 274 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 152 152 152 276 276 276 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 151 151 151 275 275 275 T T T 14 280-299 151 151 151 275 275 275 F F F 15 300-319 145 145 145 262 262 262 T T T 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 149 149 149 270 270 270 T T T 19 380-399 152 152 152 276 276 276 T T T 20 400-419 152 152 152 276 276 276 F F F 21 420-439 152 152 152 276 276 276 F F F 22 440-459 149 149 149 270 270 270 T T T 23 460-479 149 149 149 270 270 270 F F F 24 480-499 151 151 151 275 275 275 F F F 25 500-519 151 151 151 275 275 275 F F F 26 520-539 150 150 150 274 274 274 T T T 27 540-559 146 146 146 263 263 263 T T T 28 560-579 150 150 150 274 274 274 T T T 29 580-599 152 152 152 276 276 276 F F F Max gen: 152, Max flush: 276 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 152 152 152 276 276 276 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 153 153 153 277 277 277 F F F 5 100-119 153 153 153 277 277 277 F F F 6 120-139 151 151 151 275 275 275 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 150 150 150 274 274 274 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 152 152 152 276 276 276 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 153 153 153 277 277 277 F F F 14 280-299 151 151 151 275 275 275 F F F 15 300-319 153 153 153 277 277 277 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 153 153 153 277 277 277 F F F 20 400-419 152 152 152 276 276 276 F F F 21 420-439 152 152 152 276 276 276 F F F 22 440-459 153 153 153 277 277 277 F F F 23 460-479 149 149 149 270 270 270 F F F 24 480-499 151 151 151 275 275 275 F F F 25 500-519 151 151 151 275 275 275 F F F 26 520-539 153 153 153 277 277 277 F F F 27 540-559 153 153 153 277 277 277 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 152 152 152 276 276 276 F F F Max gen: 153, Max flush: 277 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.008986347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.053492272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.05454026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.054580322Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.05459494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.054609737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798} {"msg":"Crucible d93f6575-7b48-4035-891c-154cb5d97c67 has session id: da09b1c4-1e9b-4d23-854e-cee018f91347","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.05516999Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Upstairs opts: Upstairs UUID: d93f6575-7b48-4035-891c-154cb5d97c67, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.055205164Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Crucible stats registered with UUID: d93f6575-7b48-4035-891c-154cb5d97c67","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.055222101Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086417352Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798} {"msg":"d93f6575-7b48-4035-891c-154cb5d97c67 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086472353Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08649056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086521304Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086540651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086557578Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086581693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08659935Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086615878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086632975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08665876Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_connection connected from Ok(127.0.0.1:34125)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086778369Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_connection connected from Ok(127.0.0.1:38256)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086823282Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_connection connected from Ok(127.0.0.1:34652)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.086854976Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.087717056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.087759968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.087792703Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.087821278Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.087849083Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.087874919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.088414175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.088525675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.088755405Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08907518Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089099675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089114573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089191889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089252169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089279264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089297851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089312898Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[0]R flush_numbers[0..12]: [266, 276, 267, 276, 277, 277, 275, 277, 274, 274, 267, 276]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089380457Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[0]R generation[0..12]: [147, 152, 148, 152, 153, 153, 151, 153, 150, 150, 148, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089401663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08941699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[1]R flush_numbers[0..12]: [266, 276, 267, 276, 277, 277, 275, 277, 274, 274, 267, 276]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089431178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[1]R generation[0..12]: [147, 152, 148, 152, 153, 153, 151, 153, 150, 150, 148, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089445185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089463002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[2]R flush_numbers[0..12]: [266, 276, 267, 276, 277, 277, 275, 277, 274, 274, 267, 276]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08947726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[2]R generation[0..12]: [147, 152, 148, 152, 153, 153, 151, 153, 150, 150, 148, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089490927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089504825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089517823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08953093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089543798Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089556686Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"":"downstairs","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:15:51.089570204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089637942Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"0","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089654919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"1","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089668396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"client":"2","":"downstairs","session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089681664Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089694002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"d93f6575-7b48-4035-891c-154cb5d97c67 is now active with session: da09b1c4-1e9b-4d23-854e-cee018f91347","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.08970698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089721707Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798,"session_id":"da09b1c4-1e9b-4d23-854e-cee018f91347"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:51.089734275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 422 len 32768 data: 11 9 10 11 8 8 9 8 02/30 Write block 138 len 40960 data: 12 12 10 8 9 7 7 9 11 9 03/30 Write block 304 len 40960 data: 10 9 14 14 15 14 16 16 14 9 04/30 Read block 405 len 12288 05/30 Read block 421 len 24576 06/30 Write block 425 len 8192 data: 12 9 07/30 Flush 08/30 Write block 392 len 16384 data: 9 8 9 9 09/30 Flush 10/30 Write block 420 len 20480 data: 12 12 12 10 11 11/30 Flush 12/30 Write block 87 len 8192 data: 11 10 13/30 Write block 285 len 40960 data: 10 11 11 12 12 10 10 10 9 10 14/30 Write block 507 len 12288 data: 14 13 11 15/30 Read block 408 len 36864 16/30 Write block 585 len 4096 data: 11 17/30 Write block 421 len 12288 data: 13 13 11 18/30 Write block 106 len 28672 data: 14 11 10 10 11 11 12 19/30 Read block 293 len 32768 20/30 Read block 211 len 40960 21/30 Read block 204 len 36864 22/30 Write block 432 len 16384 data: 11 10 11 10 23/30 Read block 452 len 32768 24/30 Read block 401 len 4096 25/30 Read block 554 len 36864 26/30 Flush 27/30 Write block 163 len 36864 data: 16 17 14 14 13 16 15 13 14 28/30 Write block 584 len 40960 data: 10 12 8 8 8 10 12 14 16 14 29/30 Write block 221 len 16384 data: 10 10 8 9 30/30 Write block 165 len 40960 data: 15 15 14 17 16 14 15 11 11 12 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 9 Sent Done Done false 28 Acked 1027 Write 10 Sent Done Done false 29 Acked 1028 Write 4 Sent Sent Sent false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 2 2 8 Done 0 2 2 4 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:51.118705394Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 152 152 152 276 276 276 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 154 154 154 281 281 281 F F F 6 120-139 154 154 154 278 278 278 F F F 7 140-159 154 154 154 278 278 278 F F F 8 160-179 150 150 150 274 274 274 T T T 9 180-199 150 150 150 274 274 274 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 152 152 152 276 276 276 T T T 12 240-259 149 149 149 270 270 270 F F F 13 260-279 153 153 153 277 277 277 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 154 154 154 279 279 279 F F F 20 400-419 152 152 152 276 276 276 F F F 21 420-439 154 154 154 281 281 281 F F F 22 440-459 153 153 153 277 277 277 F F F 23 460-479 149 149 149 270 270 270 F F F 24 480-499 151 151 151 275 275 275 F F F 25 500-519 154 154 154 281 281 281 F F F 26 520-539 153 153 153 277 277 277 F F F 27 540-559 153 153 153 277 277 277 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 154 154 154 281 281 281 T T T Max gen: 154, Max flush: 281 A Difference in extent metadata is expected here On loop 77 Verifying data now Files /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ 167a168,169 > 15, > 15, 169,170c171 < 14, < 13, --- > 17, 172,173d172 < 15, < 13, 175,176c174 < 10, < 10, --- > 15, 177a176,177 > 11, > 12, diff found after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 152 152 152 276 276 276 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 154 154 154 281 281 281 F F F 6 120-139 154 154 154 278 278 278 F F F 7 140-159 154 154 154 278 278 278 F F F 8 160-179 155 155 155 282 282 282 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 148 148 148 267 267 267 F F F 11 220-239 155 155 155 282 282 282 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 153 153 153 277 277 277 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 154 154 154 279 279 279 F F F 20 400-419 152 152 152 276 276 276 F F F 21 420-439 154 154 154 281 281 281 F F F 22 440-459 153 153 153 277 277 277 F F F 23 460-479 149 149 149 270 270 270 F F F 24 480-499 151 151 151 275 275 275 F F F 25 500-519 154 154 154 281 281 281 F F F 26 520-539 153 153 153 277 277 277 F F F 27 540-559 153 153 153 277 277 277 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 155 155 155 282 282 282 F F F Max gen: 155, Max flush: 282 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.465134318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:55.508034452Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.508637208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.540982329Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.541868855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.541902079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.541916626Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.541931104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809} {"msg":"Crucible 6e25edff-6247-4f74-874d-70db437a1745 has session id: 6f48206f-50b4-45ca-8aad-3db8f8aba774","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.542520651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Upstairs opts: Upstairs UUID: 6e25edff-6247-4f74-874d-70db437a1745, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.542567033Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Crucible stats registered with UUID: 6e25edff-6247-4f74-874d-70db437a1745","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.542592159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573507579Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809} {"msg":"6e25edff-6247-4f74-874d-70db437a1745 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573563949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573584086Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573630657Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573665251Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573701975Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573737319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573780171Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573819955Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"io task","client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.573859748Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"io task","client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.574175903Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"io task","client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_connection connected from Ok(127.0.0.1:36189)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.574231053Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"io task","client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_connection connected from Ok(127.0.0.1:42043)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.574264757Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"io task","client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_connection connected from Ok(127.0.0.1:64300)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.574294822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"io task","client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.574809802Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.574838997Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.574857484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.57488184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.575007428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.575075976Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.57551289Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.575713345Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.575752588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576225026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576265679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576296613Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576323529Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576351294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576379559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576403835Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576426191Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[0]R flush_numbers[0..12]: [266, 276, 267, 276, 281, 281, 278, 278, 282, 274, 267, 282]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576483651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[0]R generation[0..12]: [147, 152, 148, 152, 154, 154, 154, 154, 155, 150, 148, 155]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576512776Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576528063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[1]R flush_numbers[0..12]: [266, 276, 267, 276, 281, 281, 278, 278, 282, 274, 267, 282]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576542381Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[1]R generation[0..12]: [147, 152, 148, 152, 154, 154, 154, 154, 155, 150, 148, 155]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576556798Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576570866Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[2]R flush_numbers[0..12]: [266, 276, 267, 276, 281, 281, 278, 278, 282, 274, 267, 282]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576589243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[2]R generation[0..12]: [147, 152, 148, 152, 154, 154, 154, 154, 155, 150, 148, 155]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.57660394Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576618028Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576630915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576644063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576656731Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576679857Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"":"downstairs","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576692965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576704842Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"0","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.57671837Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"1","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576731478Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"client":"2","":"downstairs","session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576744815Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576757173Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"6e25edff-6247-4f74-874d-70db437a1745 is now active with session: 6f48206f-50b4-45ca-8aad-3db8f8aba774","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576769761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576781679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809,"session_id":"6f48206f-50b4-45ca-8aad-3db8f8aba774"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:55.576796297Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 287 len 12288 data: 12 13 13 02/30 Read block 568 len 32768 03/30 Flush 04/30 Write block 70 len 32768 data: 12 14 14 10 14 14 16 10 05/30 Read block 363 len 20480 06/30 Write block 209 len 20480 data: 14 13 11 8 9 07/30 Read block 577 len 20480 08/30 Write block 117 len 20480 data: 10 14 14 11 13 09/30 Read block 328 len 16384 10/30 Read block 4 len 16384 11/30 Write block 517 len 8192 data: 16 15 12/30 Read block 28 len 40960 13/30 Flush 14/30 Write block 513 len 4096 data: 12 15/30 Flush 16/30 Flush 17/30 Flush 18/30 Write block 487 len 32768 data: 14 12 13 9 10 6 5 7 19/30 Read block 20 len 24576 20/30 Read block 1 len 16384 21/30 Read block 508 len 32768 22/30 Read block 223 len 20480 23/30 Write block 8 len 40960 data: 12 11 13 13 15 17 13 15 15 15 24/30 Read block 64 len 4096 25/30 Read block 85 len 32768 26/30 Read block 209 len 32768 27/30 Read block 407 len 8192 28/30 Write block 474 len 28672 data: 11 15 11 9 12 11 13 29/30 Read block 179 len 36864 30/30 Write block 221 len 20480 data: 11 11 9 10 9 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:15:55.602650358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1809} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 T T T 1 020-039 152 152 152 276 276 276 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 156 156 156 284 284 284 F F F 6 120-139 156 156 156 284 284 284 F F F 7 140-159 154 154 154 278 278 278 F F F 8 160-179 155 155 155 282 282 282 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 156 156 156 284 284 284 F F F 11 220-239 155 155 155 282 282 282 T T T 12 240-259 149 149 149 270 270 270 F F F 13 260-279 153 153 153 277 277 277 F F F 14 280-299 156 156 156 283 283 283 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 154 154 154 279 279 279 F F F 20 400-419 152 152 152 276 276 276 F F F 21 420-439 154 154 154 281 281 281 F F F 22 440-459 153 153 153 277 277 277 F F F 23 460-479 149 149 149 270 270 270 T T T 24 480-499 151 151 151 275 275 275 T T T 25 500-519 156 156 156 285 285 285 F F F 26 520-539 153 153 153 277 277 277 F F F 27 540-559 153 153 153 277 277 277 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 155 155 155 282 282 282 F F F Max gen: 156, Max flush: 285 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 286 286 286 F F F 1 020-039 152 152 152 276 276 276 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 154 154 154 281 281 281 F F F 5 100-119 156 156 156 284 284 284 F F F 6 120-139 156 156 156 284 284 284 F F F 7 140-159 154 154 154 278 278 278 F F F 8 160-179 155 155 155 282 282 282 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 156 156 156 284 284 284 F F F 11 220-239 157 157 157 286 286 286 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 153 153 153 277 277 277 F F F 14 280-299 156 156 156 283 283 283 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 147 147 147 266 266 266 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 154 154 154 279 279 279 F F F 20 400-419 152 152 152 276 276 276 F F F 21 420-439 154 154 154 281 281 281 F F F 22 440-459 153 153 153 277 277 277 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 157 157 157 286 286 286 F F F 25 500-519 156 156 156 285 285 285 F F F 26 520-539 153 153 153 277 277 277 F F F 27 540-559 153 153 153 277 277 277 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 155 155 155 282 282 282 F F F Max gen: 157, Max flush: 286 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.949571416Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.994004264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.994923344Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.994959988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.994979464Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.994994052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817} {"msg":"Crucible 0495679f-3ea7-4657-81a8-8e6fa3c6d1ac has session id: 0bb63ed5-182b-4e6b-8996-fec8222a6950","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.995435725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Upstairs opts: Upstairs UUID: 0495679f-3ea7-4657-81a8-8e6fa3c6d1ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.995483537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Crucible stats registered with UUID: 0495679f-3ea7-4657-81a8-8e6fa3c6d1ac","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:15:59.995511881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027258687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817} {"msg":"0495679f-3ea7-4657-81a8-8e6fa3c6d1ac active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027331914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027365798Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027409591Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027447864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027484308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027521371Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027560124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027595029Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027629443Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027672205Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_connection connected from Ok(127.0.0.1:41917)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027739493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_connection connected from Ok(127.0.0.1:35028)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027776677Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_connection connected from Ok(127.0.0.1:49748)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.027806761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.028617031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.028652784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.028680509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.028709284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.028852979Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.028895002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.029406023Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.029435018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.029450615Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030033404Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030062849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030082236Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030095493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030109411Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030122698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030134956Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030148114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[0]R flush_numbers[0..12]: [286, 276, 267, 284, 281, 284, 284, 278, 282, 274, 284, 286]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030161972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[0]R generation[0..12]: [157, 152, 148, 156, 154, 156, 156, 154, 155, 150, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030176119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030190387Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[1]R flush_numbers[0..12]: [286, 276, 267, 284, 281, 284, 284, 278, 282, 274, 284, 286]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030204134Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[1]R generation[0..12]: [157, 152, 148, 156, 154, 156, 156, 154, 155, 150, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030217982Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.03023245Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[2]R flush_numbers[0..12]: [286, 276, 267, 284, 281, 284, 284, 278, 282, 274, 284, 286]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030246517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[2]R generation[0..12]: [157, 152, 148, 156, 154, 156, 156, 154, 155, 150, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030260265Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030277562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030291079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030304247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030317195Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"Wait for a query_work_queue command to finish before sending IO } {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030341431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"":"downstairs","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030354878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030367256Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"0","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030380694Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"1","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030393991Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"client":"2","":"downstairs","session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030407609Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030420047Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"0495679f-3ea7-4657-81a8-8e6fa3c6d1ac is now active with session: 0bb63ed5-182b-4e6b-8996-fec8222a6950","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030432864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030449152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817,"session_id":"0bb63ed5-182b-4e6b-8996-fec8222a6950"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:00.030462079Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 28672 data: 16 16 15 15 9 10 8 02/30 Read block 267 len 12288 03/30 Write block 347 len 16384 data: 7 7 8 8 04/30 Read block 553 len 20480 05/30 Read block 11 len 16384 06/30 Write block 215 len 8192 data: 9 11 07/30 Flush 08/30 Flush 09/30 Write block 169 len 8192 data: 16 14 10/30 Write block 528 len 4096 data: 7 11/30 Flush 12/30 Write block 217 len 36864 data: 11 13 18 13 12 12 10 11 10 13/30 Read block 106 len 8192 14/30 Read block 100 len 4096 15/30 Write block 420 len 16384 data: 13 14 14 12 16/30 Read block 362 len 8192 17/30 Read block 278 len 20480 18/30 Read block 144 len 28672 19/30 Write block 534 len 36864 data: 12 14 12 13 13 14 15 15 14 20/30 Read block 506 len 24576 21/30 Write block 138 len 4096 data: 13 22/30 Write block 285 len 24576 data: 11 12 13 14 14 11 23/30 Write block 18 len 32768 data: 16 16 10 11 9 8 9 9 24/30 Write block 82 len 4096 data: 15 25/30 Read block 195 len 40960 26/30 Read block 279 len 36864 27/30 Read block 429 len 32768 28/30 Read block 399 len 16384 29/30 Write block 404 len 32768 data: 14 15 16 16 16 14 14 12 30/30 Read block 169 len 36864 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 12 Acked 1011 Write 9 Done Done Done false 13 Acked 1012 Read 2 Done Done Done false 14 Acked 1013 Read 1 Done Done Done false 15 Acked 1014 Write 4 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Read 7 Done Done Done false 19 Acked 1018 Write 9 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 8 Sent Done Done false 30 Acked 1029 Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 17 18 19 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:00.055382573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1817} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 287 287 287 T T T 1 020-039 158 158 158 287 287 287 T T T 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 154 154 154 281 281 281 T T T 5 100-119 156 156 156 284 284 284 F F F 6 120-139 156 156 156 284 284 284 T T T 7 140-159 154 154 154 278 278 278 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 158 158 158 287 287 287 T T T 11 220-239 157 157 157 286 286 286 T T T 12 240-259 149 149 149 270 270 270 F F F 13 260-279 153 153 153 277 277 277 F F F 14 280-299 156 156 156 283 283 283 T T T 15 300-319 154 154 154 278 278 278 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 158 158 158 287 287 287 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 154 154 154 279 279 279 F F F 20 400-419 152 152 152 276 276 276 T T T 21 420-439 154 154 154 281 281 281 T T T 22 440-459 153 153 153 277 277 277 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 157 157 157 286 286 286 F F F 25 500-519 156 156 156 285 285 285 F F F 26 520-539 158 158 158 289 289 289 T T T 27 540-559 153 153 153 277 277 277 T T T 28 560-579 153 153 153 277 277 277 F F F 29 580-599 155 155 155 282 282 282 F F F Max gen: 158, Max flush: 289 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 290 290 290 F F F 1 020-039 159 159 159 290 290 290 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 159 159 159 290 290 290 F F F 5 100-119 156 156 156 284 284 284 F F F 6 120-139 159 159 159 290 290 290 F F F 7 140-159 154 154 154 278 278 278 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 159 159 159 290 290 290 F F F 11 220-239 159 159 159 290 290 290 F F F 12 240-259 149 149 149 270 270 270 F F F 13 260-279 153 153 153 277 277 277 F F F 14 280-299 159 159 159 290 290 290 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 150 150 150 274 274 274 F F F 17 340-359 158 158 158 287 287 287 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 154 154 154 279 279 279 F F F 20 400-419 159 159 159 290 290 290 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 153 153 153 277 277 277 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 157 157 157 286 286 286 F F F 25 500-519 156 156 156 285 285 285 F F F 26 520-539 159 159 159 290 290 290 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 155 155 155 282 282 282 F F F Max gen: 159, Max flush: 290 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.465049273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.510955234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.511881433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.511919516Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.511934254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.511948721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825} {"msg":"Crucible 4b36936f-6200-430b-a944-013ffe4e4330 has session id: 64008923-25cd-4b8d-8b92-9906dcf8a79c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.512399143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Upstairs opts: Upstairs UUID: 4b36936f-6200-430b-a944-013ffe4e4330, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.512445145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Crucible stats registered with UUID: 4b36936f-6200-430b-a944-013ffe4e4330","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.51247129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543585966Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825} {"msg":"4b36936f-6200-430b-a944-013ffe4e4330 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543641276Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543671101Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543720422Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543755296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.5437916Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543825564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543857758Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543886104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"io task","client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543918588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"io task","client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543950862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"io task","client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49132)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.543990495Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"io task","client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57302)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.544027149Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"io task","client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51282)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.544143169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"io task","client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.544860064Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.544893978Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.544910285Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.544924243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.545242377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.545271073Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.545848102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.545876977Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.545892604Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546447668Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546476423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.54649122Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546508667Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546522695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546535923Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.54654793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546561038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[0]R flush_numbers[0..12]: [290, 290, 267, 284, 290, 284, 290, 278, 289, 274, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546574826Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[0]R generation[0..12]: [159, 159, 148, 156, 159, 156, 159, 154, 158, 150, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546589093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546603051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[1]R flush_numbers[0..12]: [290, 290, 267, 284, 290, 284, 290, 278, 289, 274, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546616589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[1]R generation[0..12]: [159, 159, 148, 156, 159, 156, 159, 154, 158, 150, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546630346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546644194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[2]R flush_numbers[0..12]: [290, 290, 267, 284, 290, 284, 290, 278, 289, 274, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546657851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[2]R generation[0..12]: [159, 159, 148, 156, 159, 156, 159, 154, 158, 150, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546671389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546685146Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546700544Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546713781Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546726579Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546739407Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"":"downstairs","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546752675Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:16:04.546764673Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"0","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546788888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"1","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546802146Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"client":"2","":"downstairs","session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546815674Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546828072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"4b36936f-6200-430b-a944-013ffe4e4330 is now active with session: 64008923-25cd-4b8d-8b92-9906dcf8a79c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546841049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546853067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825,"session_id":"64008923-25cd-4b8d-8b92-9906dcf8a79c"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:04.546865305Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 24576 data: 8 9 10 10 9 9 02/30 Write block 280 len 8192 data: 11 11 03/30 Write block 479 len 32768 data: 12 14 11 11 15 17 17 14 04/30 Write block 211 len 32768 data: 12 9 10 11 10 12 12 14 05/30 Read block 79 len 36864 06/30 Read block 18 len 4096 07/30 Read block 280 len 40960 08/30 Read block 585 len 4096 09/30 Read block 158 len 36864 10/30 Write block 285 len 12288 data: 12 13 14 11/30 Read block 330 len 20480 12/30 Write block 258 len 32768 data: 9 11 12 14 13 12 11 9 13/30 Read block 184 len 36864 14/30 Write block 504 len 20480 data: 13 13 14 15 14 15/30 Write block 483 len 28672 data: 16 18 18 15 15 13 14 16/30 Read block 529 len 20480 17/30 Read block 285 len 36864 18/30 Read block 422 len 36864 19/30 Flush 20/30 Write block 444 len 36864 data: 8 11 14 16 13 13 15 15 15 21/30 Read block 0 len 16384 22/30 Flush 23/30 Write block 233 len 32768 data: 11 13 14 13 13 12 13 14 24/30 Read block 87 len 24576 25/30 Write block 381 len 20480 data: 12 12 13 12 13 26/30 Flush 27/30 Write block 335 len 32768 data: 21 20 19 21 17 17 17 16 28/30 Read block 356 len 32768 29/30 Write block 270 len 20480 data: 8 9 8 8 10 30/30 Write block 494 len 24576 data: 8 7 7 11 12 12 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 5 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:04.573373662Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1825} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 290 290 290 F F F 1 020-039 159 159 159 290 290 290 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 159 159 159 290 290 290 F F F 5 100-119 156 156 156 284 284 284 F F F 6 120-139 159 159 159 290 290 290 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 160 160 160 291 291 291 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 160 160 160 291 291 291 T T T 14 280-299 160 160 160 291 291 291 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 150 150 150 274 274 274 T T T 17 340-359 158 158 158 287 287 287 T T T 18 360-379 153 153 153 277 277 277 F F F 19 380-399 160 160 160 293 293 293 F F F 20 400-419 159 159 159 290 290 290 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 160 160 160 292 292 292 F F F 23 460-479 160 160 160 291 291 291 F F F 24 480-499 160 160 160 291 291 291 T T T 25 500-519 160 160 160 291 291 291 F F F 26 520-539 159 159 159 290 290 290 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 155 155 155 282 282 282 F F F Max gen: 160, Max flush: 293 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 290 290 290 F F F 1 020-039 159 159 159 290 290 290 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 159 159 159 290 290 290 F F F 5 100-119 156 156 156 284 284 284 F F F 6 120-139 159 159 159 290 290 290 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 150 150 150 274 274 274 F F F 10 200-219 160 160 160 291 291 291 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 161 161 161 294 294 294 F F F 14 280-299 160 160 160 291 291 291 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 161 161 161 294 294 294 F F F 17 340-359 161 161 161 294 294 294 F F F 18 360-379 153 153 153 277 277 277 F F F 19 380-399 160 160 160 293 293 293 F F F 20 400-419 159 159 159 290 290 290 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 160 160 160 292 292 292 F F F 23 460-479 160 160 160 291 291 291 F F F 24 480-499 161 161 161 294 294 294 F F F 25 500-519 160 160 160 291 291 291 F F F 26 520-539 159 159 159 290 290 290 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 153 153 153 277 277 277 F F F 29 580-599 155 155 155 282 282 282 F F F Max gen: 161, Max flush: 294 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.891076695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:08.933850882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.934473174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.966812086Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.967743904Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.967781497Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.967795845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.967810662Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833} {"msg":"Crucible 1efe250d-3c0b-494d-889f-64997d530950 has session id: 906c24a8-9271-4a85-b292-a54e9c1950b8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.968251165Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Upstairs opts: Upstairs UUID: 1efe250d-3c0b-494d-889f-64997d530950, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.968302076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Crucible stats registered with UUID: 1efe250d-3c0b-494d-889f-64997d530950","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.968326822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999565856Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833} {"msg":"1efe250d-3c0b-494d-889f-64997d530950 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999619047Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999637234Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999677067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999712341Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999744435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999764301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999781698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999797976Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"io task","client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.99982851Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"io task","client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999862155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"io task","client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52748)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:08.999977914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"io task","client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:58967)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.000184179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"io task","client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54346)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.000249687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"io task","client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001006915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001043949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001067715Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001089751Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001111957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001133943Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001827193Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001862347Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.001934834Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002556935Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002584681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002599738Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002698721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002719227Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002733685Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002750452Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.00276443Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[0]R flush_numbers[0..12]: [290, 290, 267, 284, 290, 284, 290, 291, 289, 274, 291, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002794184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[0]R generation[0..12]: [159, 159, 148, 156, 159, 156, 159, 160, 158, 150, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002809182Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002823389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[1]R flush_numbers[0..12]: [290, 290, 267, 284, 290, 284, 290, 291, 289, 274, 291, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002837327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[1]R generation[0..12]: [159, 159, 148, 156, 159, 156, 159, 160, 158, 150, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002851104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002865212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[2]R flush_numbers[0..12]: [290, 290, 267, 284, 290, 284, 290, 291, 289, 274, 291, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.00287895Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[2]R generation[0..12]: [159, 159, 148, 156, 159, 156, 159, 160, 158, 150, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002892617Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002906645Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002919992Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.00293364Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002946708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002972753Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"":"downstairs","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.002999019Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.003023994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"0","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.003039172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"1","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.003052919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"client":"2","":"downstairs","session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:16:09.003075795Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.00310953Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"1efe250d-3c0b-494d-889f-64997d530950 is now active with session: 906c24a8-9271-4a85-b292-a54e9c1950b8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.003129016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.003146283Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833,"session_id":"906c24a8-9271-4a85-b292-a54e9c1950b8"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:09.003168859Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 373 len 36864 data: 15 15 16 12 12 10 11 9 13 02/30 Read block 180 len 36864 03/30 Write block 102 len 36864 data: 14 15 11 12 15 12 11 11 12 04/30 Read block 206 len 20480 05/30 Flush 06/30 Flush 07/30 Write block 534 len 20480 data: 13 15 13 14 14 08/30 Write block 105 len 40960 data: 13 16 13 12 12 13 12 13 9 11 09/30 Write block 13 len 32768 data: 18 14 16 17 17 17 17 11 10/30 Read block 106 len 36864 11/30 Write block 388 len 16384 data: 13 13 12 13 12/30 Read block 559 len 12288 13/30 Write block 271 len 36864 data: 10 9 9 11 10 12 12 11 9 14/30 Read block 242 len 4096 15/30 Flush 16/30 Write block 283 len 32768 data: 12 10 13 14 15 15 15 12 17/30 Read block 490 len 36864 18/30 Write block 385 len 8192 data: 14 14 19/30 Write block 196 len 24576 data: 14 15 15 15 14 15 20/30 Write block 88 len 12288 data: 11 10 7 21/30 Write block 206 len 36864 data: 9 11 11 15 14 13 10 11 12 22/30 Flush 23/30 Write block 339 len 16384 data: 18 18 18 17 24/30 Flush 25/30 Write block 577 len 20480 data: 13 12 14 13 12 26/30 Write block 358 len 36864 data: 6 12 12 17 20 19 20 17 19 27/30 Read block 502 len 32768 28/30 Read block 198 len 24576 29/30 Write block 100 len 20480 data: 13 13 15 16 12 30/30 Write block 455 len 40960 data: 16 19 18 15 12 13 13 14 16 18 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 5 Sent Sent Sent false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:09.032998768Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1833} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 297 297 297 F F F 1 020-039 162 162 162 297 297 297 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 162 162 162 298 298 298 F F F 5 100-119 162 162 162 297 297 297 T T T 6 120-139 159 159 159 290 290 290 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 162 162 162 298 298 298 F F F 10 200-219 162 162 162 298 298 298 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 162 162 162 297 297 297 F F F 14 280-299 162 162 162 298 298 298 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 162 162 162 299 299 299 F F F 17 340-359 162 162 162 299 299 299 T T T 18 360-379 162 162 162 295 295 295 T T T 19 380-399 162 162 162 298 298 298 F F F 20 400-419 159 159 159 290 290 290 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 160 160 160 292 292 292 T T T 23 460-479 160 160 160 291 291 291 T T T 24 480-499 161 161 161 294 294 294 F F F 25 500-519 160 160 160 291 291 291 F F F 26 520-539 162 162 162 297 297 297 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 153 153 153 277 277 277 T T T 29 580-599 155 155 155 282 282 282 T T T Max gen: 162, Max flush: 299 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 297 297 297 F F F 1 020-039 162 162 162 297 297 297 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 F F F 4 080-099 162 162 162 298 298 298 F F F 5 100-119 163 163 163 300 300 300 F F F 6 120-139 159 159 159 290 290 290 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 162 162 162 298 298 298 F F F 10 200-219 162 162 162 298 298 298 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 162 162 162 297 297 297 F F F 14 280-299 162 162 162 298 298 298 F F F 15 300-319 154 154 154 278 278 278 F F F 16 320-339 162 162 162 299 299 299 F F F 17 340-359 163 163 163 300 300 300 F F F 18 360-379 163 163 163 300 300 300 F F F 19 380-399 162 162 162 298 298 298 F F F 20 400-419 159 159 159 290 290 290 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 163 163 163 300 300 300 F F F 23 460-479 163 163 163 300 300 300 F F F 24 480-499 161 161 161 294 294 294 F F F 25 500-519 160 160 160 291 291 291 F F F 26 520-539 162 162 162 297 297 297 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 163 163 163 300 300 300 F F F 29 580-599 163 163 163 300 300 300 F F F Max gen: 163, Max flush: 300 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.415067081Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.460366818Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.461230577Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.46126825Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.461282868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.461297165Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841} {"msg":"Crucible 197fac75-2a1a-423d-b878-9f41ee7ff7f9 has session id: 9147d6ee-8f64-434b-a467-2e3db33789da","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.461708414Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Upstairs opts: Upstairs UUID: 197fac75-2a1a-423d-b878-9f41ee7ff7f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.461753856Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Crucible stats registered with UUID: 197fac75-2a1a-423d-b878-9f41ee7ff7f9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.461779202Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.492960935Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841} {"msg":"197fac75-2a1a-423d-b878-9f41ee7ff7f9 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493019125Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493038482Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493067737Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493085754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493102721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493119058Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493135705Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493151982Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"io task","client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493169419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"io task","client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493192555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"io task","client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_connection connected from Ok(127.0.0.1:50116)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493343609Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"io task","client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_connection connected from Ok(127.0.0.1:43668)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493385572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"io task","client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_connection connected from Ok(127.0.0.1:49599)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.493416116Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"io task","client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.494373609Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.494403224Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.494418432Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.494436429Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.494450586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.494506846Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495126389Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495297519Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.49540554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495774826Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495811159Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495836355Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495860801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495886316Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.495909272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496055877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496085222Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[0]R flush_numbers[0..12]: [297, 297, 267, 284, 298, 300, 290, 291, 289, 298, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496122035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[0]R generation[0..12]: [162, 162, 148, 156, 162, 163, 159, 160, 158, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496142312Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496157259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[1]R flush_numbers[0..12]: [297, 297, 267, 284, 298, 300, 290, 291, 289, 298, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496171107Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[1]R generation[0..12]: [162, 162, 148, 156, 162, 163, 159, 160, 158, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496184545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496198432Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[2]R flush_numbers[0..12]: [297, 297, 267, 284, 298, 300, 290, 291, 289, 298, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.49621338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[2]R generation[0..12]: [162, 162, 148, 156, 162, 163, 159, 160, 158, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496227327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496241275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496254162Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.49626725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496280408Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496293486Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"":"downstairs","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496306743Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496330639Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"0","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496345936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"1","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496359124Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"client":"2","":"downstairs","session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496372072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.49638431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"197fac75-2a1a-423d-b878-9f41ee7ff7f9 is now active with session: 9147d6ee-8f64-434b-a467-2e3db33789da","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496396988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496408875Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841,"session_id":"9147d6ee-8f64-434b-a467-2e3db33789da"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:13.496421033Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 368 len 12288 data: 17 15 16 03/30 Read block 194 len 12288 04/30 Write block 355 len 40960 data: 7 5 3 7 13 13 18 21 20 21 05/30 Read block 188 len 36864 06/30 Write block 516 len 16384 data: 18 17 16 12 07/30 Read block 536 len 36864 08/30 Read block 408 len 12288 09/30 Flush 10/30 Flush 11/30 Write block 573 len 20480 data: 14 14 16 14 14 12/30 Read block 50 len 36864 13/30 Read block 435 len 24576 14/30 Write block 455 len 32768 data: 17 20 19 16 13 14 14 15 15/30 Read block 329 len 16384 16/30 Write block 511 len 40960 data: 11 12 13 14 17 19 18 17 13 12 17/30 Write block 273 len 12288 data: 10 12 11 18/30 Write block 62 len 12288 data: 12 12 11 19/30 Read block 592 len 12288 20/30 Read block 374 len 24576 21/30 Write block 305 len 12288 data: 10 15 15 22/30 Read block 311 len 36864 23/30 Read block 254 len 16384 24/30 Read block 385 len 8192 25/30 Write block 504 len 40960 data: 14 14 15 16 15 12 9 12 13 14 26/30 Read block 99 len 40960 27/30 Write block 84 len 32768 data: 17 14 11 12 12 11 8 6 28/30 Read block 429 len 24576 29/30 Read block 351 len 28672 30/30 Read block 101 len 16384 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 11 Acked 1010 Write 5 Done Done Done false 12 Acked 1011 Read 9 Done Done Done false 13 Acked 1012 Read 6 Done Done Done false 14 Acked 1013 Write 8 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Write 10 Done Done Done false 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Read 3 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:13.524677066Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1841} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 297 297 297 F F F 1 020-039 162 162 162 297 297 297 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 156 156 156 284 284 284 T T T 4 080-099 162 162 162 298 298 298 T T T 5 100-119 163 163 163 300 300 300 F F F 6 120-139 159 159 159 290 290 290 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 162 162 162 298 298 298 F F F 10 200-219 162 162 162 298 298 298 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 162 162 162 297 297 297 T T T 14 280-299 162 162 162 298 298 298 F F F 15 300-319 154 154 154 278 278 278 T T T 16 320-339 162 162 162 299 299 299 F F F 17 340-359 164 164 164 302 302 302 F F F 18 360-379 164 164 164 302 302 302 F F F 19 380-399 162 162 162 298 298 298 F F F 20 400-419 159 159 159 290 290 290 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 163 163 163 300 300 300 T T T 23 460-479 163 163 163 300 300 300 T T T 24 480-499 161 161 161 294 294 294 F F F 25 500-519 164 164 164 302 302 302 T T T 26 520-539 162 162 162 297 297 297 T T T 27 540-559 159 159 159 290 290 290 F F F 28 560-579 163 163 163 300 300 300 T T T 29 580-599 163 163 163 300 300 300 F F F Max gen: 164, Max flush: 302 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 297 297 297 F F F 1 020-039 162 162 162 297 297 297 F F F 2 040-059 148 148 148 267 267 267 F F F 3 060-079 165 165 165 303 303 303 F F F 4 080-099 165 165 165 303 303 303 F F F 5 100-119 163 163 163 300 300 300 F F F 6 120-139 159 159 159 290 290 290 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 158 158 158 289 289 289 F F F 9 180-199 162 162 162 298 298 298 F F F 10 200-219 162 162 162 298 298 298 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 162 162 162 298 298 298 F F F 15 300-319 165 165 165 303 303 303 F F F 16 320-339 162 162 162 299 299 299 F F F 17 340-359 164 164 164 302 302 302 F F F 18 360-379 164 164 164 302 302 302 F F F 19 380-399 162 162 162 298 298 298 F F F 20 400-419 159 159 159 290 290 290 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 165 165 165 303 303 303 F F F 23 460-479 165 165 165 303 303 303 F F F 24 480-499 161 161 161 294 294 294 F F F 25 500-519 165 165 165 303 303 303 F F F 26 520-539 165 165 165 303 303 303 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 163 163 163 300 300 300 F F F Max gen: 165, Max flush: 303 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.905272845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.949666049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.950748681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.950800202Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.950822518Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.950846174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849} {"msg":"Crucible b3ec58a9-72a0-4e91-b364-c082c15345eb has session id: 589aa605-ecef-4892-aff6-3cfe3956c3b3","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.951263921Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Upstairs opts: Upstairs UUID: b3ec58a9-72a0-4e91-b364-c082c15345eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.951314383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Crucible stats registered with UUID: b3ec58a9-72a0-4e91-b364-c082c15345eb","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.951337988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983436293Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849} {"msg":"b3ec58a9-72a0-4e91-b364-c082c15345eb active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983499112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983531896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983653055Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983702726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983742959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983780803Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983820306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.983927257Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"io task","client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984155008Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"io task","client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984182983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"io task","client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45455)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984207648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"io task","client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:61108)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984293273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"io task","client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37343)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984333067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"io task","client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984708071Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984784418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984840698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984877132Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984913785Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.984948699Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.985401821Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.985553874Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.985584339Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986028791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986059936Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986086292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986109728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986133913Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.98615107Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986169607Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986183275Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[0]R flush_numbers[0..12]: [297, 297, 267, 303, 303, 300, 290, 291, 289, 298, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986251913Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[0]R generation[0..12]: [162, 162, 148, 165, 165, 163, 159, 160, 158, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986271889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986289826Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[1]R flush_numbers[0..12]: [297, 297, 267, 303, 303, 300, 290, 291, 289, 298, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986304054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[1]R generation[0..12]: [162, 162, 148, 165, 165, 163, 159, 160, 158, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986317951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986331919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[2]R flush_numbers[0..12]: [297, 297, 267, 303, 303, 300, 290, 291, 289, 298, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986345716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[2]R generation[0..12]: [162, 162, 148, 165, 165, 163, 159, 160, 158, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986359614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986374002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986387089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986400277Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986412895Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986425693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"":"downstairs","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986450908Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986463026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"0","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986476244Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"1","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986491241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"client":"2","":"downstairs","session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986504509Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986516866Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"b3ec58a9-72a0-4e91-b364-c082c15345eb is now active with session: 589aa605-ecef-4892-aff6-3cfe3956c3b3","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986529574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.986541622Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849,"session_id":"589aa605-ecef-4892-aff6-3cfe3956c3b3"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:17.98655366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 9 len 4096 data: 12 02/30 Write block 137 len 4096 data: 10 03/30 Read block 515 len 8192 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 289 len 8192 data: 16 13 08/30 Read block 430 len 4096 09/30 Read block 493 len 12288 10/30 Read block 185 len 4096 11/30 Read block 305 len 8192 12/30 Write block 370 len 28672 data: 17 17 17 16 16 17 13 13/30 Write block 45 len 8192 data: 12 11 14/30 Write block 168 len 40960 data: 17 17 15 15 11 11 12 10 10 10 15/30 Write block 122 len 4096 data: 12 16/30 Write block 285 len 16384 data: 14 15 16 16 17/30 Write block 471 len 8192 data: 9 11 18/30 Flush 19/30 Write block 50 len 40960 data: 9 9 9 11 11 9 9 12 12 9 20/30 Write block 397 len 20480 data: 12 11 11 12 12 21/30 Read block 371 len 12288 22/30 Write block 183 len 40960 data: 16 16 15 14 12 11 7 11 10 11 23/30 Read block 548 len 32768 24/30 Flush 25/30 Write block 167 len 28672 data: 14 18 18 16 16 12 12 26/30 Read block 550 len 28672 27/30 Read block 387 len 40960 28/30 Read block 100 len 20480 29/30 Read block 241 len 32768 30/30 Write block 248 len 28672 data: 12 11 10 10 6 7 9 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:18.010156763Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1849} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 304 304 304 F F F 1 020-039 162 162 162 297 297 297 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 165 165 165 303 303 303 F F F 4 080-099 165 165 165 303 303 303 F F F 5 100-119 163 163 163 300 300 300 F F F 6 120-139 166 166 166 307 307 307 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 166 166 166 307 307 307 T T T 9 180-199 166 166 166 308 308 308 F F F 10 200-219 162 162 162 298 298 298 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 160 160 160 293 293 293 T T T 13 260-279 165 165 165 303 303 303 F F F 14 280-299 166 166 166 307 307 307 F F F 15 300-319 165 165 165 303 303 303 F F F 16 320-339 162 162 162 299 299 299 F F F 17 340-359 164 164 164 302 302 302 F F F 18 360-379 166 166 166 307 307 307 F F F 19 380-399 166 166 166 308 308 308 F F F 20 400-419 166 166 166 308 308 308 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 165 165 165 303 303 303 F F F 23 460-479 166 166 166 307 307 307 F F F 24 480-499 161 161 161 294 294 294 F F F 25 500-519 165 165 165 303 303 303 F F F 26 520-539 165 165 165 303 303 303 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 163 163 163 300 300 300 F F F Max gen: 166, Max flush: 308 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 304 304 304 F F F 1 020-039 162 162 162 297 297 297 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 165 165 165 303 303 303 F F F 4 080-099 165 165 165 303 303 303 F F F 5 100-119 163 163 163 300 300 300 F F F 6 120-139 166 166 166 307 307 307 F F F 7 140-159 160 160 160 291 291 291 F F F 8 160-179 167 167 167 309 309 309 F F F 9 180-199 166 166 166 308 308 308 F F F 10 200-219 162 162 162 298 298 298 F F F 11 220-239 160 160 160 293 293 293 F F F 12 240-259 167 167 167 309 309 309 F F F 13 260-279 165 165 165 303 303 303 F F F 14 280-299 166 166 166 307 307 307 F F F 15 300-319 165 165 165 303 303 303 F F F 16 320-339 162 162 162 299 299 299 F F F 17 340-359 164 164 164 302 302 302 F F F 18 360-379 166 166 166 307 307 307 F F F 19 380-399 166 166 166 308 308 308 F F F 20 400-419 166 166 166 308 308 308 F F F 21 420-439 159 159 159 290 290 290 F F F 22 440-459 165 165 165 303 303 303 F F F 23 460-479 166 166 166 307 307 307 F F F 24 480-499 161 161 161 294 294 294 F F F 25 500-519 165 165 165 303 303 303 F F F 26 520-539 165 165 165 303 303 303 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 165 165 165 303 303 303 F F F 29 580-599 163 163 163 300 300 300 F F F Max gen: 167, Max flush: 309 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.296902114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:22.340585582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.34111174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.373199116Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.37392692Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.373964573Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.37398119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.373998987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857} {"msg":"Crucible 4754b215-b288-4769-ac83-e8062a316473 has session id: 5ec64d58-c2bc-4b14-981a-d59666021d94","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.374492671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Upstairs opts: Upstairs UUID: 4754b215-b288-4769-ac83-e8062a316473, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.374540743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Crucible stats registered with UUID: 4754b215-b288-4769-ac83-e8062a316473","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.374565419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405226783Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857} {"msg":"4754b215-b288-4769-ac83-e8062a316473 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405272745Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405292302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405335334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405369878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405411391Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405447935Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405483209Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405520112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"io task","client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405547477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"io task","client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405606787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"io task","client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_connection connected from Ok(127.0.0.1:53003)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405640001Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"io task","client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_connection connected from Ok(127.0.0.1:35582)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405670906Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"io task","client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_connection connected from Ok(127.0.0.1:33774)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.405802793Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"io task","client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.40639365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.406464308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.406498702Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.406530146Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.406559241Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.406588726Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407153548Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.40719602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407300862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407748734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407783828Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407838329Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407861045Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407875752Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.40788927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407901768Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407915215Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[0]R flush_numbers[0..12]: [304, 297, 308, 303, 303, 300, 307, 291, 309, 308, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.40794346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[0]R generation[0..12]: [166, 162, 166, 165, 165, 163, 166, 160, 167, 166, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407958058Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407972185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[1]R flush_numbers[0..12]: [304, 297, 308, 303, 303, 300, 307, 291, 309, 308, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.407985863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[1]R generation[0..12]: [166, 162, 166, 165, 165, 163, 166, 160, 167, 166, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.40800295Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408017477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[2]R flush_numbers[0..12]: [304, 297, 308, 303, 303, 300, 307, 291, 309, 308, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408031325Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[2]R generation[0..12]: [166, 162, 166, 165, 165, 163, 166, 160, 167, 166, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408045043Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.40805885Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408071808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408085006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408097754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408110611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"":"downstairs","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408123929Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408135857Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"0","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:16:22.408149265Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"1","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408207414Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"client":"2","":"downstairs","session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.40823212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408245928Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"4754b215-b288-4769-ac83-e8062a316473 is now active with session: 5ec64d58-c2bc-4b14-981a-d59666021d94","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408262395Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:22.408275112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857,"session_id":"5ec64d58-c2bc-4b14-981a-d59666021d94"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:16:22.40828761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 228 len 12288 data: 9 9 9 04/30 Read block 328 len 20480 05/30 Flush 06/30 Write block 224 len 20480 data: 12 11 10 9 10 07/30 Flush 08/30 Write block 468 len 36864 data: 11 8 6 10 12 9 12 16 12 09/30 Read block 270 len 32768 10/30 Read block 522 len 40960 11/30 Write block 137 len 28672 data: 11 14 13 11 9 10 8 12/30 Write block 569 len 40960 data: 8 7 9 13 15 15 17 15 15 13 13/30 Read block 551 len 8192 14/30 Read block 151 len 20480 15/30 Read block 422 len 28672 16/30 Write block 473 len 20480 data: 10 13 17 13 10 17/30 Write block 434 len 32768 data: 12 11 7 7 6 7 4 4 18/30 Write block 399 len 16384 data: 12 13 13 14 19/30 Write block 272 len 28672 data: 10 11 13 12 13 13 12 20/30 Read block 576 len 8192 21/30 Read block 525 len 20480 22/30 Read block 67 len 4096 23/30 Write block 211 len 16384 data: 14 11 12 13 24/30 Read block 496 len 24576 25/30 Write block 90 len 28672 data: 9 7 7 6 11 11 13 26/30 Read block 470 len 28672 27/30 Write block 24 len 28672 data: 10 10 7 6 5 8 8 28/30 Write block 492 len 36864 data: 7 6 9 8 8 12 13 13 10 29/30 Write block 106 len 32768 data: 17 14 13 13 14 13 14 10 30/30 Write block 350 len 24576 data: 9 6 7 8 7 8 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 8 Acked 1007 Write 9 Done Done Done false 9 Acked 1008 Read 8 Done Done Done false 10 Acked 1009 Read 10 Done Done Done false 11 Acked 1010 Write 7 Done Done Done false 12 Acked 1011 Write 10 Done Done Done false 13 Acked 1012 Read 2 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Read 7 Done Done Done false 16 Acked 1015 Write 5 Done Done Done false 17 Acked 1016 Write 8 Done Done Done false 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:22.43202474Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1857} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 304 304 304 F F F 1 020-039 162 162 162 297 297 297 T T T 2 040-059 166 166 166 308 308 308 F F F 3 060-079 165 165 165 303 303 303 F F F 4 080-099 165 165 165 303 303 303 T T T 5 100-119 163 163 163 300 300 300 T T T 6 120-139 166 166 166 307 307 307 T T T 7 140-159 160 160 160 291 291 291 T T T 8 160-179 167 167 167 309 309 309 F F F 9 180-199 166 166 166 308 308 308 F F F 10 200-219 162 162 162 298 298 298 T T T 11 220-239 168 168 168 313 313 313 F F F 12 240-259 167 167 167 309 309 309 F F F 13 260-279 165 165 165 303 303 303 T T T 14 280-299 166 166 166 307 307 307 F F F 15 300-319 165 165 165 303 303 303 F F F 16 320-339 162 162 162 299 299 299 F F F 17 340-359 164 164 164 302 302 302 T T T 18 360-379 166 166 166 307 307 307 F F F 19 380-399 166 166 166 308 308 308 T T T 20 400-419 166 166 166 308 308 308 T T T 21 420-439 159 159 159 290 290 290 T T T 22 440-459 165 165 165 303 303 303 T T T 23 460-479 166 166 166 307 307 307 T T T 24 480-499 161 161 161 294 294 294 T T T 25 500-519 165 165 165 303 303 303 T T T 26 520-539 165 165 165 303 303 303 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 165 165 165 303 303 303 T T T 29 580-599 163 163 163 300 300 300 F F F Max gen: 168, Max flush: 313 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 304 304 304 F F F 1 020-039 169 169 169 314 314 314 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 165 165 165 303 303 303 F F F 4 080-099 169 169 169 314 314 314 F F F 5 100-119 169 169 169 314 314 314 F F F 6 120-139 169 169 169 314 314 314 F F F 7 140-159 169 169 169 314 314 314 F F F 8 160-179 167 167 167 309 309 309 F F F 9 180-199 166 166 166 308 308 308 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 168 168 168 313 313 313 F F F 12 240-259 167 167 167 309 309 309 F F F 13 260-279 169 169 169 314 314 314 F F F 14 280-299 166 166 166 307 307 307 F F F 15 300-319 165 165 165 303 303 303 F F F 16 320-339 162 162 162 299 299 299 F F F 17 340-359 169 169 169 314 314 314 F F F 18 360-379 166 166 166 307 307 307 F F F 19 380-399 169 169 169 314 314 314 F F F 20 400-419 169 169 169 314 314 314 F F F 21 420-439 169 169 169 314 314 314 F F F 22 440-459 169 169 169 314 314 314 F F F 23 460-479 169 169 169 314 314 314 F F F 24 480-499 169 169 169 314 314 314 F F F 25 500-519 169 169 169 314 314 314 F F F 26 520-539 165 165 165 303 303 303 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 169 169 169 314 314 314 F F F 29 580-599 163 163 163 300 300 300 F F F Max gen: 169, Max flush: 314 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.942391176Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.987068611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.98799194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.988038292Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.988056189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.988074116Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865} {"msg":"Crucible a2ebd57f-fb51-4b8e-89d9-61b393953879 has session id: a7caddeb-041f-4f1b-b2b0-6bb065460ff9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.988648336Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Upstairs opts: Upstairs UUID: a2ebd57f-fb51-4b8e-89d9-61b393953879, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.988694298Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Crucible stats registered with UUID: a2ebd57f-fb51-4b8e-89d9-61b393953879","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:26.988719743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.019956827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865} {"msg":"a2ebd57f-fb51-4b8e-89d9-61b393953879 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020014728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020047032Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020094714Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020130367Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.02017201Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020206794Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020240208Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020275002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020310106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.02034628Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40485)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020381914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51415)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020512261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37182)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.020545865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.021115896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.021146311Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.021161878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.021175606Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.021318561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.021350025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.021856847Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022006661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022072559Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022440945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022481528Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022510213Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022569063Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022597948Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022613035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022637281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022652009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[0]R flush_numbers[0..12]: [304, 314, 308, 303, 314, 314, 314, 314, 309, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022717457Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[0]R generation[0..12]: [166, 169, 166, 165, 169, 169, 169, 169, 167, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022738723Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022753431Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[1]R flush_numbers[0..12]: [304, 314, 308, 303, 314, 314, 314, 314, 309, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022767418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[1]R generation[0..12]: [166, 169, 166, 165, 169, 169, 169, 169, 167, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022785505Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022799963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[2]R flush_numbers[0..12]: [304, 314, 308, 303, 314, 314, 314, 314, 309, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.02281383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[2]R generation[0..12]: [166, 169, 166, 165, 169, 169, 169, 169, 167, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022827878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022841865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022854903Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022868231Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022881229Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:16:27.022894136Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"":"downstairs","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022915163Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022927321Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"0","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022940788Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"1","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022953716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"client":"2","":"downstairs","session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022966914Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022979102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"a2ebd57f-fb51-4b8e-89d9-61b393953879 is now active with session: a7caddeb-041f-4f1b-b2b0-6bb065460ff9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.022993739Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.023005877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865,"session_id":"a7caddeb-041f-4f1b-b2b0-6bb065460ff9"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:27.023018115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 404 len 36864 data: 15 16 17 17 17 15 15 13 13 02/30 Write block 495 len 12288 data: 9 9 13 03/30 Read block 134 len 16384 04/30 Write block 27 len 32768 data: 7 6 9 9 10 9 11 14 05/30 Write block 433 len 40960 data: 11 13 12 8 8 7 8 5 5 6 06/30 Flush 07/30 Write block 261 len 8192 data: 15 14 08/30 Read block 352 len 8192 09/30 Read block 374 len 32768 10/30 Flush 11/30 Flush 12/30 Write block 481 len 16384 data: 12 12 17 19 13/30 Write block 34 len 4096 data: 15 14/30 Write block 488 len 36864 data: 14 15 10 11 8 7 10 10 10 15/30 Write block 452 len 20480 data: 16 16 18 18 21 16/30 Write block 485 len 32768 data: 19 16 16 15 16 11 12 9 17/30 Write block 102 len 8192 data: 16 17 18/30 Write block 582 len 36864 data: 11 14 11 13 9 9 9 11 13 19/30 Write block 147 len 20480 data: 10 8 9 10 11 20/30 Write block 576 len 20480 data: 16 16 14 15 14 21/30 Write block 76 len 40960 data: 17 11 10 14 15 14 16 16 18 15 22/30 Write block 68 len 40960 data: 13 14 13 15 15 11 15 15 18 12 23/30 Flush 24/30 Write block 319 len 32768 data: 6 7 6 6 5 7 8 8 25/30 Read block 443 len 16384 26/30 Flush 27/30 Write block 495 len 28672 data: 11 11 14 14 14 11 14 28/30 Write block 396 len 16384 data: 10 13 12 13 29/30 Read block 270 len 36864 30/30 Read block 440 len 8192 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:27.05103271Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1865} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 304 304 304 F F F 1 020-039 170 170 170 318 318 318 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 170 170 170 318 318 318 F F F 4 080-099 170 170 170 318 318 318 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 169 169 169 314 314 314 F F F 7 140-159 170 170 170 318 318 318 F F F 8 160-179 167 167 167 309 309 309 F F F 9 180-199 166 166 166 308 308 308 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 168 168 168 313 313 313 F F F 12 240-259 167 167 167 309 309 309 F F F 13 260-279 170 170 170 316 316 316 F F F 14 280-299 166 166 166 307 307 307 F F F 15 300-319 170 170 170 319 319 319 F F F 16 320-339 170 170 170 319 319 319 F F F 17 340-359 169 169 169 314 314 314 F F F 18 360-379 166 166 166 307 307 307 F F F 19 380-399 169 169 169 314 314 314 T T T 20 400-419 170 170 170 315 315 315 F F F 21 420-439 170 170 170 315 315 315 F F F 22 440-459 170 170 170 318 318 318 F F F 23 460-479 169 169 169 314 314 314 F F F 24 480-499 170 170 170 318 318 318 T T T 25 500-519 169 169 169 314 314 314 T T T 26 520-539 165 165 165 303 303 303 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 170 170 170 318 318 318 F F F 29 580-599 170 170 170 318 318 318 F F F Max gen: 170, Max flush: 319 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 304 304 304 F F F 1 020-039 170 170 170 318 318 318 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 170 170 170 318 318 318 F F F 4 080-099 170 170 170 318 318 318 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 169 169 169 314 314 314 F F F 7 140-159 170 170 170 318 318 318 F F F 8 160-179 167 167 167 309 309 309 F F F 9 180-199 166 166 166 308 308 308 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 168 168 168 313 313 313 F F F 12 240-259 167 167 167 309 309 309 F F F 13 260-279 170 170 170 316 316 316 F F F 14 280-299 166 166 166 307 307 307 F F F 15 300-319 170 170 170 319 319 319 F F F 16 320-339 170 170 170 319 319 319 F F F 17 340-359 169 169 169 314 314 314 F F F 18 360-379 166 166 166 307 307 307 F F F 19 380-399 171 171 171 320 320 320 F F F 20 400-419 170 170 170 315 315 315 F F F 21 420-439 170 170 170 315 315 315 F F F 22 440-459 170 170 170 318 318 318 F F F 23 460-479 169 169 169 314 314 314 F F F 24 480-499 171 171 171 320 320 320 F F F 25 500-519 171 171 171 320 320 320 F F F 26 520-539 165 165 165 303 303 303 F F F 27 540-559 159 159 159 290 290 290 F F F 28 560-579 170 170 170 318 318 318 F F F 29 580-599 170 170 170 318 318 318 F F F Max gen: 171, Max flush: 320 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.356765596Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.402345515Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.403211984Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.403249547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.403268044Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.403283211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873} {"msg":"Crucible 70a6f91b-d407-4a99-a068-789c3080b2b7 has session id: 59d910ec-2347-42e5-8829-22f41ec330c4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.403739452Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Upstairs opts: Upstairs UUID: 70a6f91b-d407-4a99-a068-789c3080b2b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.403788693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Crucible stats registered with UUID: 70a6f91b-d407-4a99-a068-789c3080b2b7","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.403814009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.434968158Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873} {"msg":"70a6f91b-d407-4a99-a068-789c3080b2b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.43500949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435027177Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435055642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435087497Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435120661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435154965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435189439Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435224823Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435254118Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435271905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:59787)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435396493Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63555)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435440185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34006)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.435491296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.436238916Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.436267332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.436281939Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.436295216Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.436412236Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.436451859Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.436983177Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.43702266Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437143419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437581343Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437612667Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437628564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437641372Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.43765501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437668388Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437688014Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437701882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[0]R flush_numbers[0..12]: [304, 318, 308, 318, 318, 318, 314, 318, 309, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437765461Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[0]R generation[0..12]: [166, 170, 166, 170, 170, 170, 169, 170, 167, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437786667Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437801365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[1]R flush_numbers[0..12]: [304, 318, 308, 318, 318, 318, 314, 318, 309, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437815272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[1]R generation[0..12]: [166, 170, 166, 170, 170, 170, 169, 170, 167, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.43782902Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437842917Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[2]R flush_numbers[0..12]: [304, 318, 308, 318, 318, 318, 314, 318, 309, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437856375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[2]R generation[0..12]: [166, 170, 166, 170, 170, 170, 169, 170, 167, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437872942Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437887139Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437900137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437913285Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437926123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437938861Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437963886Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.437975934Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"0","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.438006199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"1","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.438021716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"client":"2","":"downstairs","session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.438035104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.438047392Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"70a6f91b-d407-4a99-a068-789c3080b2b7 is now active with session: 59d910ec-2347-42e5-8829-22f41ec330c4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.438060169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.438072067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873,"session_id":"59d910ec-2347-42e5-8829-22f41ec330c4"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:31.438084315Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 571 len 40960 data: 10 14 16 16 18 17 17 15 16 15 02/30 Write block 14 len 40960 data: 15 17 18 18 18 18 12 12 10 9 03/30 Read block 147 len 36864 04/30 Write block 480 len 28672 data: 15 13 13 18 20 20 17 05/30 Read block 388 len 24576 06/30 Read block 462 len 20480 07/30 Write block 374 len 36864 data: 17 18 14 13 11 12 10 14 13 08/30 Write block 166 len 32768 data: 15 15 19 19 17 17 13 13 09/30 Read block 161 len 28672 10/30 Write block 264 len 36864 data: 12 10 10 10 8 9 9 11 11 11/30 Read block 270 len 20480 12/30 Write block 557 len 32768 data: 15 12 12 10 11 13 11 11 13/30 Read block 413 len 32768 14/30 Read block 278 len 40960 15/30 Read block 443 len 36864 16/30 Write block 4 len 28672 data: 9 12 14 15 13 13 14 17/30 Write block 438 len 12288 data: 8 9 6 18/30 Read block 469 len 8192 19/30 Read block 14 len 32768 20/30 Write block 534 len 12288 data: 14 16 14 21/30 Flush 22/30 Write block 282 len 36864 data: 12 13 11 15 16 17 17 17 14 23/30 Write block 86 len 24576 data: 12 13 13 12 10 8 24/30 Write block 322 len 4096 data: 7 25/30 Flush 26/30 Write block 161 len 12288 data: 11 14 17 27/30 Write block 524 len 28672 data: 13 11 7 5 8 12 13 28/30 Write block 280 len 4096 data: 12 29/30 Read block 165 len 16384 30/30 Write block 165 len 20480 data: 15 16 16 20 20 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:31.464305982Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1873} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 172 172 172 321 321 321 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 170 170 170 318 318 318 F F F 4 080-099 172 172 172 322 322 322 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 169 169 169 314 314 314 F F F 7 140-159 170 170 170 318 318 318 F F F 8 160-179 172 172 172 321 321 321 T T T 9 180-199 166 166 166 308 308 308 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 168 168 168 313 313 313 F F F 12 240-259 167 167 167 309 309 309 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 172 172 172 322 322 322 T T T 15 300-319 170 170 170 319 319 319 F F F 16 320-339 172 172 172 322 322 322 F F F 17 340-359 169 169 169 314 314 314 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 172 172 172 321 321 321 F F F 20 400-419 170 170 170 315 315 315 F F F 21 420-439 172 172 172 321 321 321 F F F 22 440-459 172 172 172 321 321 321 F F F 23 460-479 169 169 169 314 314 314 F F F 24 480-499 172 172 172 321 321 321 F F F 25 500-519 171 171 171 320 320 320 F F F 26 520-539 172 172 172 321 321 321 T T T 27 540-559 172 172 172 321 321 321 F F F 28 560-579 172 172 172 321 321 321 F F F 29 580-599 172 172 172 321 321 321 F F F Max gen: 172, Max flush: 322 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 172 172 172 321 321 321 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 170 170 170 318 318 318 F F F 4 080-099 172 172 172 322 322 322 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 169 169 169 314 314 314 F F F 7 140-159 170 170 170 318 318 318 F F F 8 160-179 173 173 173 323 323 323 F F F 9 180-199 166 166 166 308 308 308 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 168 168 168 313 313 313 F F F 12 240-259 167 167 167 309 309 309 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 173 173 173 323 323 323 F F F 15 300-319 170 170 170 319 319 319 F F F 16 320-339 172 172 172 322 322 322 F F F 17 340-359 169 169 169 314 314 314 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 172 172 172 321 321 321 F F F 20 400-419 170 170 170 315 315 315 F F F 21 420-439 172 172 172 321 321 321 F F F 22 440-459 172 172 172 321 321 321 F F F 23 460-479 169 169 169 314 314 314 F F F 24 480-499 172 172 172 321 321 321 F F F 25 500-519 171 171 171 320 320 320 F F F 26 520-539 173 173 173 323 323 323 F F F 27 540-559 172 172 172 321 321 321 F F F 28 560-579 172 172 172 321 321 321 F F F 29 580-599 172 172 172 321 321 321 F F F Max gen: 173, Max flush: 323 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.759198245Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:35.80278Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.803343893Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.835321858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.836093003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.836126928Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.836140715Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.836159442Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881} {"msg":"Crucible 0396566d-908c-4877-a2ec-4d35276ee2e9 has session id: 1cfed232-1943-478b-a676-5f6a405077b8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.836679991Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Upstairs opts: Upstairs UUID: 0396566d-908c-4877-a2ec-4d35276ee2e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.836725533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Crucible stats registered with UUID: 0396566d-908c-4877-a2ec-4d35276ee2e9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.83674671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.867883661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881} {"msg":"0396566d-908c-4877-a2ec-4d35276ee2e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.867940681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868019498Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.86806421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868092665Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868115051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868135827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868157574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.86817879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"io task","client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868201806Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"io task","client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868224092Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"io task","client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65060)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868271344Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"io task","client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38914)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868298899Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"io task","client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:50921)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.868324255Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"io task","client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.86926954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.869299965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.869315152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.869337738Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.869364733Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.869390879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.869987325Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870142808Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870182081Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870602888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870642101Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870668687Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870694152Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870709639Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870723247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870792115Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870812232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[0]R flush_numbers[0..12]: [321, 321, 308, 318, 322, 318, 314, 318, 323, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870835817Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[0]R generation[0..12]: [172, 172, 166, 170, 172, 170, 169, 170, 173, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870850625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870864852Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[1]R flush_numbers[0..12]: [321, 321, 308, 318, 322, 318, 314, 318, 323, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.87087863Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[1]R generation[0..12]: [172, 172, 166, 170, 172, 170, 169, 170, 173, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870892218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870906035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[2]R flush_numbers[0..12]: [321, 321, 308, 318, 322, 318, 314, 318, 323, 308, 314, 313]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870919553Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[2]R generation[0..12]: [172, 172, 166, 170, 172, 170, 169, 170, 173, 166, 169, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870933131Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870946868Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870963385Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870976773Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.870990491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871003388Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"":"downstairs","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871016386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871028264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"0","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871041212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"1","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871054199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"client":"2","":"downstairs","session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871085084Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.87110915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"0396566d-908c-4877-a2ec-4d35276ee2e9 is now active with session: 1cfed232-1943-478b-a676-5f6a405077b8","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871135185Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871158991Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881,"session_id":"1cfed232-1943-478b-a676-5f6a405077b8"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:35.871183247Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 542 len 12288 data: 15 12 12 02/30 Write block 510 len 20480 data: 10 13 14 15 15 03/30 Write block 380 len 8192 data: 11 15 04/30 Write block 404 len 32768 data: 16 17 18 18 18 16 16 14 05/30 Write block 247 len 24576 data: 9 13 12 11 11 7 06/30 Write block 439 len 12288 data: 10 7 6 07/30 Write block 220 len 4096 data: 14 08/30 Write block 178 len 36864 data: 10 9 11 11 15 17 17 16 15 09/30 Read block 357 len 40960 10/30 Write block 438 len 36864 data: 9 11 8 7 7 7 9 12 15 11/30 Read block 315 len 8192 12/30 Read block 542 len 4096 13/30 Write block 323 len 12288 data: 6 8 9 14/30 Write block 475 len 28672 data: 18 14 11 13 13 16 14 15/30 Read block 575 len 8192 16/30 Write block 574 len 32768 data: 17 19 18 18 16 17 16 13 17/30 Read block 510 len 24576 18/30 Read block 39 len 32768 19/30 Write block 526 len 36864 data: 8 6 9 13 14 10 10 12 15 20/30 Flush 21/30 Write block 125 len 40960 data: 14 12 14 10 8 9 8 10 8 9 22/30 Read block 61 len 28672 23/30 Read block 229 len 32768 24/30 Read block 23 len 40960 25/30 Write block 565 len 40960 data: 12 10 11 10 9 8 11 15 17 18 26/30 Flush 27/30 Write block 85 len 40960 data: 16 13 14 14 13 11 9 8 7 12 28/30 Write block 591 len 20480 data: 15 17 15 13 10 29/30 Read block 247 len 24576 30/30 Write block 535 len 32768 data: 17 15 15 15 15 16 16 16 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 5 Done Done Sent false 29 Acked 1028 Read 6 Done Done Sent false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 3 3 1 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:35.896695887Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1881} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 172 172 172 321 321 321 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 170 170 170 318 318 318 F F F 4 080-099 172 172 172 322 322 322 T T T 5 100-119 170 170 170 318 318 318 F F F 6 120-139 174 174 174 325 325 325 F F F 7 140-159 170 170 170 318 318 318 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 174 174 174 324 324 324 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 174 174 174 324 324 324 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 173 173 173 323 323 323 F F F 15 300-319 170 170 170 319 319 319 F F F 16 320-339 174 174 174 324 324 324 F F F 17 340-359 169 169 169 314 314 314 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 174 174 174 324 324 324 F F F 20 400-419 174 174 174 324 324 324 F F F 21 420-439 174 174 174 324 324 324 F F F 22 440-459 174 174 174 324 324 324 F F F 23 460-479 174 174 174 324 324 324 F F F 24 480-499 174 174 174 324 324 324 F F F 25 500-519 174 174 174 324 324 324 F F F 26 520-539 174 174 174 324 324 324 T T T 27 540-559 174 174 174 324 324 324 T T T 28 560-579 174 174 174 325 325 325 F F F 29 580-599 174 174 174 324 324 324 T T T Max gen: 174, Max flush: 325 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 172 172 172 321 321 321 F F F 2 040-059 166 166 166 308 308 308 F F F 3 060-079 170 170 170 318 318 318 F F F 4 080-099 175 175 175 326 326 326 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 174 174 174 325 325 325 F F F 7 140-159 170 170 170 318 318 318 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 174 174 174 324 324 324 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 174 174 174 324 324 324 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 173 173 173 323 323 323 F F F 15 300-319 170 170 170 319 319 319 F F F 16 320-339 174 174 174 324 324 324 F F F 17 340-359 169 169 169 314 314 314 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 174 174 174 324 324 324 F F F 20 400-419 174 174 174 324 324 324 F F F 21 420-439 174 174 174 324 324 324 F F F 22 440-459 174 174 174 324 324 324 F F F 23 460-479 174 174 174 324 324 324 F F F 24 480-499 174 174 174 324 324 324 F F F 25 500-519 174 174 174 324 324 324 F F F 26 520-539 175 175 175 326 326 326 F F F 27 540-559 175 175 175 326 326 326 F F F 28 560-579 174 174 174 325 325 325 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 175, Max flush: 326 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.246792014Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.29140341Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.292633267Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.29267122Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.292685567Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.292699855Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889} {"msg":"Crucible 65247e2f-9bca-4ae3-959c-e4c261f71e92 has session id: 0d3a1d80-edf2-408e-93cf-593614655e93","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.293260538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Upstairs opts: Upstairs UUID: 65247e2f-9bca-4ae3-959c-e4c261f71e92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.29330595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Crucible stats registered with UUID: 65247e2f-9bca-4ae3-959c-e4c261f71e92","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.293329506Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324519628Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889} {"msg":"65247e2f-9bca-4ae3-959c-e4c261f71e92 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324569199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324587156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.32462326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324656094Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324690168Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324725052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324758086Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.3247913Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"io task","client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324827834Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"io task","client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324860488Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"io task","client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_connection connected from Ok(127.0.0.1:55381)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324892503Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"io task","client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_connection connected from Ok(127.0.0.1:60498)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324922977Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"io task","client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_connection connected from Ok(127.0.0.1:59096)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.324953442Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"io task","client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.325691984Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.325720059Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.325734716Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.325748734Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.325882961Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.325914635Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.326456561Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.326486476Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.326646348Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327130583Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327157979Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327172386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327184954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327198301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327211719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327224037Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327241754Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[0]R flush_numbers[0..12]: [321, 321, 308, 318, 326, 318, 325, 318, 324, 324, 314, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327303343Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[0]R generation[0..12]: [172, 172, 166, 170, 175, 170, 174, 170, 174, 174, 169, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327325879Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327340677Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[1]R flush_numbers[0..12]: [321, 321, 308, 318, 326, 318, 325, 318, 324, 324, 314, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327354514Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[1]R generation[0..12]: [172, 172, 166, 170, 175, 170, 174, 170, 174, 174, 169, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327368082Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.32738172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[2]R flush_numbers[0..12]: [321, 321, 308, 318, 326, 318, 325, 318, 324, 324, 314, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327395427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[2]R generation[0..12]: [172, 172, 166, 170, 175, 170, 174, 170, 174, 174, 169, 174]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327409145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327422833Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.32743554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327448538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327461306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327474564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"":"downstairs","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327502149Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327514477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"0","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327527864Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"1","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327540782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"client":"2","":"downstairs","session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.3275539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327566088Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"65247e2f-9bca-4ae3-959c-e4c261f71e92 is now active with session: 0d3a1d80-edf2-408e-93cf-593614655e93","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327578865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327590743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889,"session_id":"0d3a1d80-edf2-408e-93cf-593614655e93"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:40.327602711Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 417 len 12288 data: 15 11 12 02/30 Write block 334 len 32768 data: 18 22 21 20 22 19 19 19 03/30 Write block 295 len 24576 data: 8 9 10 8 5 7 04/30 Read block 231 len 32768 05/30 Read block 152 len 24576 06/30 Read block 576 len 16384 07/30 Write block 63 len 36864 data: 13 12 13 15 13 14 15 14 16 08/30 Read block 179 len 4096 09/30 Write block 381 len 8192 data: 16 14 10/30 Write block 220 len 16384 data: 15 13 13 11 11/30 Flush 12/30 Write block 138 len 16384 data: 15 14 12 10 13/30 Write block 560 len 28672 data: 11 12 14 12 12 13 11 14/30 Write block 335 len 40960 data: 23 22 21 23 20 20 20 18 15 13 15/30 Read block 217 len 16384 16/30 Write block 546 len 4096 data: 13 17/30 Read block 402 len 36864 18/30 Read block 8 len 8192 19/30 Read block 582 len 4096 20/30 Read block 554 len 36864 21/30 Write block 39 len 24576 data: 11 12 15 13 14 12 22/30 Read block 185 len 8192 23/30 Write block 409 len 20480 data: 17 17 15 14 14 24/30 Read block 392 len 16384 25/30 Read block 593 len 8192 26/30 Write block 574 len 4096 data: 19 27/30 Read block 240 len 28672 28/30 Read block 340 len 4096 29/30 Read block 194 len 24576 30/30 Read block 141 len 24576 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 12 Acked 1011 Write 4 Done Done Done false 13 Acked 1012 Write 7 Done Done Done false 14 Acked 1013 Write 10 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Write 1 Done Done Done false 17 Acked 1016 Read 9 Done Done Done false 18 Acked 1017 Read 2 Done Done Done false 19 Acked 1018 Read 1 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 19 19 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:40.349350617Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1889} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 172 172 172 321 321 321 T T T 2 040-059 166 166 166 308 308 308 T T T 3 060-079 176 176 176 327 327 327 F F F 4 080-099 175 175 175 326 326 326 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 174 174 174 325 325 325 T T T 7 140-159 170 170 170 318 318 318 T T T 8 160-179 174 174 174 324 324 324 F F F 9 180-199 174 174 174 324 324 324 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 176 176 176 327 327 327 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 176 176 176 327 327 327 F F F 15 300-319 176 176 176 327 327 327 F F F 16 320-339 176 176 176 327 327 327 T T T 17 340-359 176 176 176 327 327 327 T T T 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 176 176 176 327 327 327 T T T 21 420-439 174 174 174 324 324 324 F F F 22 440-459 174 174 174 324 324 324 F F F 23 460-479 174 174 174 324 324 324 F F F 24 480-499 174 174 174 324 324 324 F F F 25 500-519 174 174 174 324 324 324 F F F 26 520-539 175 175 175 326 326 326 F F F 27 540-559 175 175 175 326 326 326 T T T 28 560-579 174 174 174 325 325 325 T T T 29 580-599 175 175 175 326 326 326 F F F Max gen: 176, Max flush: 327 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 177 177 177 328 328 328 F F F 2 040-059 177 177 177 328 328 328 F F F 3 060-079 176 176 176 327 327 327 F F F 4 080-099 175 175 175 326 326 326 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 177 177 177 328 328 328 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 174 174 174 324 324 324 F F F 10 200-219 169 169 169 314 314 314 F F F 11 220-239 176 176 176 327 327 327 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 176 176 176 327 327 327 F F F 15 300-319 176 176 176 327 327 327 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 177 177 177 328 328 328 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 177 177 177 328 328 328 F F F 21 420-439 174 174 174 324 324 324 F F F 22 440-459 174 174 174 324 324 324 F F F 23 460-479 174 174 174 324 324 324 F F F 24 480-499 174 174 174 324 324 324 F F F 25 500-519 174 174 174 324 324 324 F F F 26 520-539 175 175 175 326 326 326 F F F 27 540-559 177 177 177 328 328 328 F F F 28 560-579 177 177 177 328 328 328 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 177, Max flush: 328 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.730069134Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.774344219Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.775218547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.77525629Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.775271018Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.775285495Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897} {"msg":"Crucible 5e6fc3c1-82e3-419d-a638-b5bac2421eb5 has session id: c70cd558-5c39-4409-87c7-1e98cd9f19ee","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.775736547Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Upstairs opts: Upstairs UUID: 5e6fc3c1-82e3-419d-a638-b5bac2421eb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.775789188Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Crucible stats registered with UUID: 5e6fc3c1-82e3-419d-a638-b5bac2421eb5","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.775820742Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.806809919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897} {"msg":"5e6fc3c1-82e3-419d-a638-b5bac2421eb5 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.806887125Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.806922129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.806966582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807000166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.8070334Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807061665Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807081302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807103848Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"io task","client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807136602Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"io task","client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807168916Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"io task","client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:53808)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807239094Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"io task","client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:63160)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807590153Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"io task","client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:60315)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.807619028Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"io task","client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808113692Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808147276Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808163093Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808176791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808302009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808337263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808894696Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.808936949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.8089842Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809524117Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.80956146Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809588725Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809607682Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809622089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809635827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809648105Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809661522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[0]R flush_numbers[0..12]: [321, 328, 328, 327, 326, 318, 328, 328, 324, 324, 314, 327]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.80967951Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[0]R generation[0..12]: [172, 177, 177, 176, 175, 170, 177, 177, 174, 174, 169, 176]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809694067Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809708114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[1]R flush_numbers[0..12]: [321, 328, 328, 327, 326, 318, 328, 328, 324, 324, 314, 327]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809722172Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[1]R generation[0..12]: [172, 177, 177, 176, 175, 170, 177, 177, 174, 174, 169, 176]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.80973589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809749837Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[2]R flush_numbers[0..12]: [321, 328, 328, 327, 326, 318, 328, 328, 324, 324, 314, 327]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809763435Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[2]R generation[0..12]: [172, 177, 177, 176, 175, 170, 177, 177, 174, 174, 169, 176]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809776913Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809795269Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809819235Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809901931Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809930556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809950462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"":"downstairs","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.80996453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.809981257Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"0","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.810003933Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"1","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.810026179Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"client":"2","":"downstairs","session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.810041307Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:16:44.810056964Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"5e6fc3c1-82e3-419d-a638-b5bac2421eb5 is now active with session: c70cd558-5c39-4409-87c7-1e98cd9f19ee","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.810092827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.810109065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897,"session_id":"c70cd558-5c39-4409-87c7-1e98cd9f19ee"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:44.810121913Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 419 len 32768 data: 13 14 15 15 13 12 13 10 02/30 Read block 516 len 8192 03/30 Write block 232 len 4096 data: 9 04/30 Read block 51 len 24576 05/30 Read block 18 len 16384 06/30 Read block 353 len 8192 07/30 Read block 101 len 8192 08/30 Read block 133 len 40960 09/30 Write block 514 len 4096 data: 16 10/30 Read block 211 len 36864 11/30 Read block 250 len 16384 12/30 Read block 461 len 36864 13/30 Write block 124 len 16384 data: 17 15 13 15 14/30 Write block 358 len 8192 data: 8 14 15/30 Read block 327 len 8192 16/30 Read block 397 len 8192 17/30 Read block 283 len 40960 18/30 Write block 75 len 12288 data: 16 19 13 19/30 Write block 198 len 32768 data: 16 16 15 16 15 13 13 8 20/30 Write block 498 len 24576 data: 15 15 12 15 15 13 21/30 Read block 88 len 40960 22/30 Read block 95 len 8192 23/30 Read block 264 len 12288 24/30 Read block 242 len 16384 25/30 Write block 415 len 40960 data: 16 14 16 12 14 15 16 16 14 13 26/30 Write block 414 len 4096 data: 14 27/30 Write block 350 len 24576 data: 10 7 8 9 8 9 28/30 Read block 289 len 40960 29/30 Read block 83 len 40960 30/30 Write block 52 len 36864 data: 10 12 12 10 10 13 13 10 10 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 8 Done Done Done false 2 Acked 1001 Read 2 Done Done Done false 3 Acked 1002 Write 1 Done Done Done false 4 Acked 1003 Read 6 Done Done Done false 5 Acked 1004 Read 4 Done Done Done false 6 Acked 1005 Read 2 Done Done Done false 7 Acked 1006 Read 2 Done Done Done false 8 Acked 1007 Read 10 Done Done Done false 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Read 9 Done Done Done false 11 Acked 1010 Read 4 Done Done Done false 12 Acked 1011 Read 9 Done Done Done false 13 Acked 1012 Write 4 Done Done Done false 14 Acked 1013 Write 2 Done Done Done false 15 Acked 1014 Read 2 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Write 6 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:44.829988965Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1897} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 177 177 177 328 328 328 F F F 2 040-059 177 177 177 328 328 328 T T T 3 060-079 176 176 176 327 327 327 T T T 4 080-099 175 175 175 326 326 326 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 177 177 177 328 328 328 T T T 7 140-159 177 177 177 328 328 328 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 174 174 174 324 324 324 T T T 10 200-219 169 169 169 314 314 314 T T T 11 220-239 176 176 176 327 327 327 T T T 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 176 176 176 327 327 327 F F F 15 300-319 176 176 176 327 327 327 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 177 177 177 328 328 328 T T T 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 177 177 177 328 328 328 T T T 21 420-439 174 174 174 324 324 324 T T T 22 440-459 174 174 174 324 324 324 F F F 23 460-479 174 174 174 324 324 324 F F F 24 480-499 174 174 174 324 324 324 T T T 25 500-519 174 174 174 324 324 324 T T T 26 520-539 175 175 175 326 326 326 F F F 27 540-559 177 177 177 328 328 328 F F F 28 560-579 177 177 177 328 328 328 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 177, Max flush: 328 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 321 321 321 F F F 1 020-039 177 177 177 328 328 328 F F F 2 040-059 178 178 178 329 329 329 F F F 3 060-079 178 178 178 329 329 329 F F F 4 080-099 175 175 175 326 326 326 F F F 5 100-119 170 170 170 318 318 318 F F F 6 120-139 178 178 178 329 329 329 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 178 178 178 329 329 329 F F F 10 200-219 178 178 178 329 329 329 F F F 11 220-239 178 178 178 329 329 329 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 176 176 176 327 327 327 F F F 15 300-319 176 176 176 327 327 327 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 178 178 178 329 329 329 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 178 178 178 329 329 329 F F F 21 420-439 178 178 178 329 329 329 F F F 22 440-459 174 174 174 324 324 324 F F F 23 460-479 174 174 174 324 324 324 F F F 24 480-499 178 178 178 329 329 329 F F F 25 500-519 178 178 178 329 329 329 F F F 26 520-539 175 175 175 326 326 326 F F F 27 540-559 177 177 177 328 328 328 F F F 28 560-579 177 177 177 328 328 328 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 178, Max flush: 329 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.23571555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.279691827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.280506106Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.28054155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.280555697Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.280569915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905} {"msg":"Crucible ab5b2037-db28-482b-b827-07953fba6cb0 has session id: f2fd3580-707c-48a0-adeb-d46fad922dbd","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.281121029Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Upstairs opts: Upstairs UUID: ab5b2037-db28-482b-b827-07953fba6cb0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.281159312Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Crucible stats registered with UUID: ab5b2037-db28-482b-b827-07953fba6cb0","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.281180148Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.311879896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905} {"msg":"ab5b2037-db28-482b-b827-07953fba6cb0 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.311943035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.311976619Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312021731Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312105267Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312171145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312306742Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312356513Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312386388Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"io task","client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312415183Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"io task","client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312443288Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"io task","client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33596)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312472413Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"io task","client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36747)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312504617Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"io task","client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:56716)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.312539821Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"io task","client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.313042814Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.313079377Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.313184539Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.31323924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.313561963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.313603156Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.31363795Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.313852433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314264051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314468176Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314490942Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314505419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314517827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314531235Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314544273Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314872845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314909469Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[0]R flush_numbers[0..12]: [321, 328, 329, 329, 326, 318, 329, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314934174Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[0]R generation[0..12]: [172, 177, 178, 178, 175, 170, 178, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.31496014Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.314985546Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[1]R flush_numbers[0..12]: [321, 328, 329, 329, 326, 318, 329, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315011321Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[1]R generation[0..12]: [172, 177, 178, 178, 175, 170, 178, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315034797Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315058623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[2]R flush_numbers[0..12]: [321, 328, 329, 329, 326, 318, 329, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315083939Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[2]R generation[0..12]: [172, 177, 178, 178, 175, 170, 178, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315109414Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.31513501Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Max found gen is 179","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315157036Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Generation requested: 180 >= found:179","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315184261Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Next flush: 330","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-09-27T17:16:49.315223284Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315258338Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"":"downstairs","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315287203Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315312049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"0","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315339114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"1","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315366189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"client":"2","":"downstairs","session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315393325Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315423349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"ab5b2037-db28-482b-b827-07953fba6cb0 is now active with session: f2fd3580-707c-48a0-adeb-d46fad922dbd","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315450575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315474031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905,"session_id":"f2fd3580-707c-48a0-adeb-d46fad922dbd"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:49.315489358Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 124 len 16384 data: 18 16 14 16 02/30 Write block 58 len 8192 data: 14 11 03/30 Read block 206 len 36864 04/30 Write block 18 len 12288 data: 19 19 13 05/30 Write block 80 len 32768 data: 16 15 17 17 19 17 14 15 06/30 Write block 284 len 8192 data: 12 16 07/30 Write block 426 len 20480 data: 11 9 10 9 10 08/30 Write block 110 len 32768 data: 15 14 15 11 12 7 10 11 09/30 Read block 72 len 28672 10/30 Read block 499 len 16384 11/30 Read block 204 len 4096 12/30 Read block 351 len 28672 13/30 Read block 142 len 16384 14/30 Write block 520 len 40960 data: 13 13 14 12 14 12 9 7 10 14 15/30 Read block 479 len 36864 16/30 Read block 332 len 8192 17/30 Read block 359 len 16384 18/30 Write block 114 len 24576 data: 13 8 11 12 15 15 19/30 Read block 240 len 36864 20/30 Read block 568 len 8192 21/30 Read block 236 len 36864 22/30 Flush 23/30 Write block 544 len 36864 data: 13 11 14 11 11 13 16 17 18 24/30 Write block 545 len 12288 data: 12 15 12 25/30 Read block 320 len 4096 26/30 Write block 524 len 4096 data: 15 27/30 Write block 0 len 4096 data: 3 28/30 Write block 467 len 36864 data: 11 12 9 7 11 13 11 14 19 29/30 Read block 362 len 4096 30/30 Read block 73 len 40960 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:49.338713427Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1905} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 330 330 330 T T T 1 020-039 180 180 180 330 330 330 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 178 178 178 329 329 329 F F F 4 080-099 180 180 180 330 330 330 F F F 5 100-119 180 180 180 330 330 330 F F F 6 120-139 180 180 180 330 330 330 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 178 178 178 329 329 329 F F F 10 200-219 178 178 178 329 329 329 F F F 11 220-239 178 178 178 329 329 329 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 180 180 180 330 330 330 F F F 15 300-319 176 176 176 327 327 327 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 178 178 178 329 329 329 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 178 178 178 329 329 329 F F F 21 420-439 180 180 180 330 330 330 F F F 22 440-459 174 174 174 324 324 324 F F F 23 460-479 174 174 174 324 324 324 T T T 24 480-499 178 178 178 329 329 329 F F F 25 500-519 178 178 178 329 329 329 F F F 26 520-539 180 180 180 330 330 330 T T T 27 540-559 177 177 177 328 328 328 T T T 28 560-579 177 177 177 328 328 328 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 180, Max flush: 330 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 331 331 331 F F F 1 020-039 180 180 180 330 330 330 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 178 178 178 329 329 329 F F F 4 080-099 180 180 180 330 330 330 F F F 5 100-119 180 180 180 330 330 330 F F F 6 120-139 180 180 180 330 330 330 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 178 178 178 329 329 329 F F F 10 200-219 178 178 178 329 329 329 F F F 11 220-239 178 178 178 329 329 329 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 F F F 14 280-299 180 180 180 330 330 330 F F F 15 300-319 176 176 176 327 327 327 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 178 178 178 329 329 329 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 178 178 178 329 329 329 F F F 21 420-439 180 180 180 330 330 330 F F F 22 440-459 174 174 174 324 324 324 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 178 178 178 329 329 329 F F F 25 500-519 178 178 178 329 329 329 F F F 26 520-539 181 181 181 331 331 331 F F F 27 540-559 181 181 181 331 331 331 F F F 28 560-579 177 177 177 328 328 328 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 181, Max flush: 331 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.657886494Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:53.701256077Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.701860332Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.734029664Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.735047267Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.735091999Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.735109076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.735126883Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913} {"msg":"Crucible 5d1a982d-42d3-4355-bb0c-9de33cfcd78b has session id: 96c03a72-2a68-4b0f-bb20-493c3e89b243","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.735617918Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Upstairs opts: Upstairs UUID: 5d1a982d-42d3-4355-bb0c-9de33cfcd78b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.735672468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Crucible stats registered with UUID: 5d1a982d-42d3-4355-bb0c-9de33cfcd78b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.735701023Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767218938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913} {"msg":"5d1a982d-42d3-4355-bb0c-9de33cfcd78b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767275618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767292586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.76732392Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767341957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767359354Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767376211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767393458Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767409905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"io task","client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767427102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"io task","client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767443949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"io task","client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_connection connected from Ok(127.0.0.1:54735)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767579056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"io task","client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_connection connected from Ok(127.0.0.1:57291)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767629937Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"io task","client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_connection connected from Ok(127.0.0.1:38387)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.767780741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"io task","client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.768443335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.76847455Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.768613116Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.768673945Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.768800653Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.768822829Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769128816Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769303896Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.76944949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769765055Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769789671Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769804418Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769861719Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769881875Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.769895802Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770062704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.77008222Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[0]R flush_numbers[0..12]: [331, 330, 330, 329, 330, 330, 330, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770111145Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[0]R generation[0..12]: [181, 180, 180, 178, 180, 180, 180, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770126882Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.77014114Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[1]R flush_numbers[0..12]: [331, 330, 330, 329, 330, 330, 330, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770155047Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[1]R generation[0..12]: [181, 180, 180, 178, 180, 180, 180, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770168635Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770182723Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[2]R flush_numbers[0..12]: [331, 330, 330, 329, 330, 330, 330, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.77019746Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[2]R generation[0..12]: [181, 180, 180, 178, 180, 180, 180, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770211148Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770224905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770237713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770250911Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770263658Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770276496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"":"downstairs","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770289634Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:16:53.770303582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"0","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770325188Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"1","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770338445Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"client":"2","":"downstairs","session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770351713Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770364011Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"5d1a982d-42d3-4355-bb0c-9de33cfcd78b is now active with session: 96c03a72-2a68-4b0f-bb20-493c3e89b243","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770377869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770389927Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913,"session_id":"96c03a72-2a68-4b0f-bb20-493c3e89b243"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:53.770403384Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 70 len 4096 data: 15 03/30 Write block 415 len 32768 data: 17 15 17 13 15 16 17 17 04/30 Flush 05/30 Write block 573 len 8192 data: 18 20 06/30 Write block 111 len 4096 data: 15 07/30 Read block 380 len 32768 08/30 Read block 332 len 24576 09/30 Read block 14 len 12288 10/30 Write block 86 len 28672 data: 15 16 15 14 12 10 9 11/30 Read block 122 len 36864 12/30 Flush 13/30 Write block 314 len 4096 data: 8 14/30 Read block 313 len 40960 15/30 Flush 16/30 Write block 515 len 4096 data: 18 17/30 Read block 215 len 20480 18/30 Read block 430 len 24576 19/30 Read block 133 len 36864 20/30 Write block 116 len 12288 data: 12 13 16 21/30 Read block 499 len 32768 22/30 Write block 411 len 4096 data: 16 23/30 Write block 538 len 20480 data: 16 16 17 17 17 24/30 Write block 456 len 8192 data: 22 20 25/30 Read block 270 len 12288 26/30 Read block 110 len 24576 27/30 Read block 299 len 8192 28/30 Write block 270 len 4096 data: 10 29/30 Write block 547 len 24576 data: 13 12 14 17 18 19 30/30 Write block 430 len 20480 data: 11 10 12 12 14 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 1 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Read 6 Done Done Done false 19 Acked 1018 Read 9 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:53.794054588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1913} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 331 331 331 F F F 1 020-039 180 180 180 330 330 330 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 182 182 182 333 333 333 F F F 4 080-099 182 182 182 334 334 334 F F F 5 100-119 182 182 182 334 334 334 T T T 6 120-139 180 180 180 330 330 330 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 178 178 178 329 329 329 F F F 10 200-219 178 178 178 329 329 329 F F F 11 220-239 178 178 178 329 329 329 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 172 172 172 321 321 321 T T T 14 280-299 180 180 180 330 330 330 F F F 15 300-319 182 182 182 335 335 335 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 178 178 178 329 329 329 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 182 182 182 333 333 333 T T T 21 420-439 182 182 182 333 333 333 T T T 22 440-459 174 174 174 324 324 324 T T T 23 460-479 181 181 181 331 331 331 F F F 24 480-499 178 178 178 329 329 329 F F F 25 500-519 178 178 178 329 329 329 T T T 26 520-539 181 181 181 331 331 331 T T T 27 540-559 181 181 181 331 331 331 T T T 28 560-579 182 182 182 334 334 334 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 182, Max flush: 335 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 331 331 331 F F F 1 020-039 180 180 180 330 330 330 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 182 182 182 333 333 333 F F F 4 080-099 182 182 182 334 334 334 F F F 5 100-119 183 183 183 336 336 336 F F F 6 120-139 180 180 180 330 330 330 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 174 174 174 324 324 324 F F F 9 180-199 178 178 178 329 329 329 F F F 10 200-219 178 178 178 329 329 329 F F F 11 220-239 178 178 178 329 329 329 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 183 183 183 336 336 336 F F F 14 280-299 180 180 180 330 330 330 F F F 15 300-319 182 182 182 335 335 335 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 178 178 178 329 329 329 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 183 183 183 336 336 336 F F F 21 420-439 183 183 183 336 336 336 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 178 178 178 329 329 329 F F F 25 500-519 183 183 183 336 336 336 F F F 26 520-539 183 183 183 336 336 336 F F F 27 540-559 183 183 183 336 336 336 F F F 28 560-579 182 182 182 334 334 334 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 183, Max flush: 336 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.190557109Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.234790721Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.238550597Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.238616346Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.238642221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.238669816Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921} {"msg":"Crucible a94ab41f-cec2-4fea-9e9e-82a7c372a494 has session id: cd1149c0-b977-46c1-b5f5-053c76f26f7a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.239179708Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Upstairs opts: Upstairs UUID: a94ab41f-cec2-4fea-9e9e-82a7c372a494, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.239231068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Crucible stats registered with UUID: a94ab41f-cec2-4fea-9e9e-82a7c372a494","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.239256364Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270431049Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921} {"msg":"a94ab41f-cec2-4fea-9e9e-82a7c372a494 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.27048175Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270497798Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270525293Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.27054143Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270555268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270569635Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270583413Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.27059663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"io task","client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270612587Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"io task","client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270635934Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"io task","client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64844)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270739036Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"io task","client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37197)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270776579Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"io task","client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63767)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.270831569Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"io task","client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.271652897Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.271682221Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.271698099Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.271712006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.271929489Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.271967782Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.272534533Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.272563508Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.272635775Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273202127Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273228962Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.27324341Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273256438Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273270285Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273283673Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273295921Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273309298Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[0]R flush_numbers[0..12]: [331, 330, 330, 333, 334, 336, 330, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273323076Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[0]R generation[0..12]: [181, 180, 180, 182, 182, 183, 180, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273337013Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.27335528Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[1]R flush_numbers[0..12]: [331, 330, 330, 333, 334, 336, 330, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273369688Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[1]R generation[0..12]: [181, 180, 180, 182, 182, 183, 180, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273383575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273397453Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[2]R flush_numbers[0..12]: [331, 330, 330, 333, 334, 336, 330, 328, 324, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273411051Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[2]R generation[0..12]: [181, 180, 180, 182, 182, 183, 180, 177, 174, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273424618Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273438216Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273451264Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273464741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273477659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273490527Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"":"downstairs","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273503524Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273515482Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"0","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273543588Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"1","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273556595Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"client":"2","":"downstairs","session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273572052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.27358443Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"a94ab41f-cec2-4fea-9e9e-82a7c372a494 is now active with session: cd1149c0-b977-46c1-b5f5-053c76f26f7a","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273597178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273609256Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921,"session_id":"cd1149c0-b977-46c1-b5f5-053c76f26f7a"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:16:58.273621484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 124 len 8192 data: 19 17 02/30 Read block 153 len 4096 03/30 Write block 539 len 8192 data: 17 18 04/30 Write block 197 len 28672 data: 16 17 17 16 17 16 14 05/30 Read block 573 len 8192 06/30 Write block 344 len 32768 data: 14 8 9 8 8 9 11 8 07/30 Read block 267 len 36864 08/30 Flush 09/30 Write block 490 len 24576 data: 12 13 10 8 11 12 10/30 Flush 11/30 Write block 220 len 40960 data: 16 14 14 12 13 12 11 10 11 10 12/30 Write block 37 len 8192 data: 10 11 13/30 Flush 14/30 Write block 66 len 24576 data: 16 14 15 16 16 17 15/30 Write block 168 len 4096 data: 21 16/30 Read block 463 len 40960 17/30 Write block 189 len 36864 data: 8 12 11 12 9 10 12 15 17 18/30 Read block 514 len 36864 19/30 Write block 132 len 16384 data: 11 9 10 8 20/30 Read block 68 len 8192 21/30 Read block 428 len 24576 22/30 Read block 291 len 40960 23/30 Flush 24/30 Write block 165 len 12288 data: 16 17 17 25/30 Write block 198 len 28672 data: 18 18 17 18 17 15 14 26/30 Flush 27/30 Write block 430 len 12288 data: 12 11 13 28/30 Read block 403 len 4096 29/30 Write block 504 len 20480 data: 15 15 16 17 16 30/30 Read block 132 len 12288 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 5 Done Done Sent false 30 Acked 1029 Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 3 4 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:16:58.30260541Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1921} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 331 331 331 F F F 1 020-039 184 184 184 339 339 339 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 184 184 184 340 340 340 F F F 4 080-099 182 182 182 334 334 334 F F F 5 100-119 183 183 183 336 336 336 F F F 6 120-139 184 184 184 340 340 340 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 184 184 184 341 341 341 F F F 11 220-239 184 184 184 339 339 339 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 183 183 183 336 336 336 F F F 14 280-299 180 180 180 330 330 330 F F F 15 300-319 182 182 182 335 335 335 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 183 183 183 336 336 336 F F F 21 420-439 183 183 183 336 336 336 T T T 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 184 184 184 338 338 338 F F F 25 500-519 183 183 183 336 336 336 T T T 26 520-539 184 184 184 337 337 337 F F F 27 540-559 184 184 184 337 337 337 F F F 28 560-579 182 182 182 334 334 334 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 184, Max flush: 341 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 331 331 331 F F F 1 020-039 184 184 184 339 339 339 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 184 184 184 340 340 340 F F F 4 080-099 182 182 182 334 334 334 F F F 5 100-119 183 183 183 336 336 336 F F F 6 120-139 184 184 184 340 340 340 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 184 184 184 341 341 341 F F F 11 220-239 184 184 184 339 339 339 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 183 183 183 336 336 336 F F F 14 280-299 180 180 180 330 330 330 F F F 15 300-319 182 182 182 335 335 335 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 F F F 20 400-419 183 183 183 336 336 336 F F F 21 420-439 185 185 185 342 342 342 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 184 184 184 338 338 338 F F F 25 500-519 185 185 185 342 342 342 F F F 26 520-539 184 184 184 337 337 337 F F F 27 540-559 184 184 184 337 337 337 F F F 28 560-579 182 182 182 334 334 334 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 185, Max flush: 342 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.587271003Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.631856094Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.63285777Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.632905872Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.632923538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.632941135Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929} {"msg":"Crucible 002783a5-7169-4954-a601-3233cba932c2 has session id: 07dcddb1-fa14-4486-a826-2f6c444ec62e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.633455596Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Upstairs opts: Upstairs UUID: 002783a5-7169-4954-a601-3233cba932c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.633513356Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Crucible stats registered with UUID: 002783a5-7169-4954-a601-3233cba932c2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.633536742Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664681562Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929} {"msg":"002783a5-7169-4954-a601-3233cba932c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664737142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664754349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664782354Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664799491Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664816558Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664833216Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664855801Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664872619Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"io task","client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664889926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"io task","client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.664907212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"io task","client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48318)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.665024142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"io task","client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48416)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.665065485Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"io task","client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49187)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.66509423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"io task","client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.66589564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.665924036Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.665938423Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.665952211Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.665994663Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.66601301Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.666615375Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.666658218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.666731565Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.66727674Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667306065Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667321972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.66733526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667349158Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667362625Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667375023Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667388421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[0]R flush_numbers[0..12]: [331, 339, 330, 340, 334, 336, 340, 328, 341, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667430224Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[0]R generation[0..12]: [181, 184, 180, 184, 182, 183, 184, 177, 184, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.66744889Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667463228Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[1]R flush_numbers[0..12]: [331, 339, 330, 340, 334, 336, 340, 328, 341, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667476905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[1]R generation[0..12]: [181, 184, 180, 184, 182, 183, 184, 177, 184, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667490383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667504481Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[2]R flush_numbers[0..12]: [331, 339, 330, 340, 334, 336, 340, 328, 341, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667521308Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[2]R generation[0..12]: [181, 184, 180, 184, 182, 183, 184, 177, 184, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667535155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667548853Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667561781Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667575428Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Next flush: 343","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667588446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667601624Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"":"downstairs","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667615611Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667627999Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"Wait for a query_work_queue command to finish before sending IO 0","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667653235Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"1","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667666612Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"client":"2","":"downstairs","session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.66768004Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667692448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"002783a5-7169-4954-a601-3233cba932c2 is now active with session: 07dcddb1-fa14-4486-a826-2f6c444ec62e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667705376Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667717323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929,"session_id":"07dcddb1-fa14-4486-a826-2f6c444ec62e"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:02.667733021Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 123 len 12288 data: 14 20 18 02/30 Read block 105 len 8192 03/30 Write block 28 len 32768 data: 7 10 10 11 10 12 16 11 04/30 Write block 228 len 28672 data: 12 11 10 9 10 12 14 05/30 Read block 312 len 24576 06/30 Write block 217 len 24576 data: 13 15 19 17 15 15 07/30 Flush 08/30 Write block 81 len 24576 data: 16 18 18 20 18 16 09/30 Write block 109 len 16384 data: 14 16 16 16 10/30 Read block 463 len 32768 11/30 Read block 221 len 16384 12/30 Flush 13/30 Write block 399 len 8192 data: 14 14 14/30 Write block 523 len 36864 data: 13 16 13 10 8 11 15 15 11 15/30 Write block 288 len 12288 data: 18 18 15 16/30 Read block 185 len 16384 17/30 Read block 6 len 8192 18/30 Read block 160 len 36864 19/30 Read block 173 len 20480 20/30 Read block 385 len 4096 21/30 Write block 226 len 36864 data: 12 11 13 12 11 10 11 13 15 22/30 Read block 0 len 8192 23/30 Read block 423 len 32768 24/30 Read block 252 len 16384 25/30 Write block 550 len 20480 data: 18 19 20 16 14 26/30 Read block 64 len 24576 27/30 Write block 513 len 16384 data: 16 17 19 20 28/30 Read block 553 len 8192 29/30 Write block 37 len 12288 data: 11 12 12 30/30 Read block 461 len 8192 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 2 Done Done Done false 14 Acked 1013 Write 9 Done Done Done false 15 Acked 1014 Write 3 Done Done Done false 16 Acked 1015 Read 4 Done Done Done false 17 Acked 1016 Read 2 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:02.691942468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1929} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 331 331 331 F F F 1 020-039 186 186 186 343 343 343 T T T 2 040-059 180 180 180 330 330 330 F F F 3 060-079 184 184 184 340 340 340 F F F 4 080-099 186 186 186 344 344 344 F F F 5 100-119 186 186 186 344 344 344 F F F 6 120-139 186 186 186 343 343 343 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 186 186 186 343 343 343 F F F 11 220-239 186 186 186 343 343 343 T T T 12 240-259 174 174 174 324 324 324 F F F 13 260-279 183 183 183 336 336 336 F F F 14 280-299 180 180 180 330 330 330 T T T 15 300-319 182 182 182 335 335 335 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 176 176 176 327 327 327 T T T 20 400-419 183 183 183 336 336 336 T T T 21 420-439 185 185 185 342 342 342 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 184 184 184 338 338 338 F F F 25 500-519 185 185 185 342 342 342 T T T 26 520-539 184 184 184 337 337 337 T T T 27 540-559 184 184 184 337 337 337 T T T 28 560-579 182 182 182 334 334 334 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 186, Max flush: 344 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 331 331 331 F F F 1 020-039 187 187 187 345 345 345 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 184 184 184 340 340 340 F F F 4 080-099 186 186 186 344 344 344 F F F 5 100-119 186 186 186 344 344 344 F F F 6 120-139 186 186 186 343 343 343 F F F 7 140-159 177 177 177 328 328 328 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 186 186 186 343 343 343 F F F 11 220-239 187 187 187 345 345 345 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 183 183 183 336 336 336 F F F 14 280-299 187 187 187 345 345 345 F F F 15 300-319 182 182 182 335 335 335 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 187 187 187 345 345 345 F F F 20 400-419 187 187 187 345 345 345 F F F 21 420-439 185 185 185 342 342 342 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 184 184 184 338 338 338 F F F 25 500-519 187 187 187 345 345 345 F F F 26 520-539 187 187 187 345 345 345 F F F 27 540-559 187 187 187 345 345 345 F F F 28 560-579 182 182 182 334 334 334 F F F 29 580-599 175 175 175 326 326 326 F F F Max gen: 187, Max flush: 345 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.066950089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:07.110249764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.110833922Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.142571119Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.143323638Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.143356852Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.14337038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.143384698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937} {"msg":"Crucible 7be53921-a22c-464d-b716-93da705e8a6e has session id: abf60387-e53c-4f64-bcf0-531b1bc21582","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.143787318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Upstairs opts: Upstairs UUID: 7be53921-a22c-464d-b716-93da705e8a6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.14383189Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Crucible stats registered with UUID: 7be53921-a22c-464d-b716-93da705e8a6e","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.143856556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.174985959Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937} {"msg":"7be53921-a22c-464d-b716-93da705e8a6e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175023642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175042479Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175070574Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175088421Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175105488Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175122025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175145121Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175162058Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175179335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175196192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_connection connected from Ok(127.0.0.1:46253)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175290526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_connection connected from Ok(127.0.0.1:47748)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.175339447Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_connection connected from Ok(127.0.0.1:34256)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.17537926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.176284582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.176314457Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.176330454Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.176344642Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.17635787Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.176379336Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.176905634Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177052989Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177080144Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177494412Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177531865Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177548582Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.17762326Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177643556Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177657463Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177828614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.17784877Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[0]R flush_numbers[0..12]: [331, 345, 330, 340, 344, 344, 343, 328, 341, 341, 343, 345]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177873286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[0]R generation[0..12]: [181, 187, 180, 184, 186, 186, 186, 177, 184, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177888433Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177902681Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[1]R flush_numbers[0..12]: [331, 345, 330, 340, 344, 344, 343, 328, 341, 341, 343, 345]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177916669Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[1]R generation[0..12]: [181, 187, 180, 184, 186, 186, 186, 177, 184, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177930506Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177944704Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[2]R flush_numbers[0..12]: [331, 345, 330, 340, 344, 344, 343, 328, 341, 341, 343, 345]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.17796197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[2]R generation[0..12]: [181, 187, 180, 184, 186, 186, 186, 177, 184, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177976228Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.177990396Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178003763Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178016981Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178030009Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178042827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"":"downstairs","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178055974Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178069302Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"0","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178095178Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"1","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178108315Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"client":"2","":"downstairs","session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178121563Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178133741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"7be53921-a22c-464d-b716-93da705e8a6e is now active with session: abf60387-e53c-4f64-bcf0-531b1bc21582","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178146549Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178158446Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937,"session_id":"abf60387-e53c-4f64-bcf0-531b1bc21582"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:07.178173434Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 121 len 4096 data: 14 02/30 Write block 586 len 28672 data: 10 10 10 12 14 16 18 03/30 Read block 484 len 4096 04/30 Write block 319 len 4096 data: 7 05/30 Write block 395 len 8192 data: 10 11 06/30 Write block 62 len 32768 data: 13 14 13 14 17 15 16 17 07/30 Flush 08/30 Write block 9 len 28672 data: 14 15 14 16 19 16 18 09/30 Write block 3 len 16384 data: 5 10 13 15 10/30 Read block 180 len 12288 11/30 Read block 508 len 4096 12/30 Read block 97 len 20480 13/30 Read block 249 len 16384 14/30 Read block 34 len 16384 15/30 Write block 210 len 20480 data: 15 15 12 13 14 16/30 Flush 17/30 Write block 261 len 16384 data: 16 15 13 13 18/30 Read block 232 len 16384 19/30 Flush 20/30 Write block 480 len 4096 data: 17 21/30 Read block 223 len 24576 22/30 Read block 274 len 28672 23/30 Read block 225 len 36864 24/30 Flush 25/30 Write block 223 len 28672 data: 13 14 13 13 12 14 13 26/30 Read block 569 len 4096 27/30 Read block 417 len 8192 28/30 Write block 550 len 16384 data: 19 20 21 17 29/30 Read block 573 len 8192 30/30 Write block 139 len 28672 data: 15 13 11 11 9 8 10 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:07.203362141Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1937} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 347 347 347 F F F 1 020-039 187 187 187 345 345 345 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 188 188 188 346 346 346 F F F 4 080-099 186 186 186 344 344 344 F F F 5 100-119 186 186 186 344 344 344 F F F 6 120-139 188 188 188 346 346 346 T T T 7 140-159 177 177 177 328 328 328 T T T 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 187 187 187 345 345 345 T T T 12 240-259 174 174 174 324 324 324 F F F 13 260-279 188 188 188 348 348 348 F F F 14 280-299 187 187 187 345 345 345 F F F 15 300-319 188 188 188 346 346 346 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 188 188 188 346 346 346 F F F 20 400-419 187 187 187 345 345 345 F F F 21 420-439 185 185 185 342 342 342 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 188 188 188 349 349 349 F F F 25 500-519 187 187 187 345 345 345 F F F 26 520-539 187 187 187 345 345 345 F F F 27 540-559 187 187 187 345 345 345 T T T 28 560-579 182 182 182 334 334 334 F F F 29 580-599 188 188 188 346 346 346 F F F Max gen: 188, Max flush: 349 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 347 347 347 F F F 1 020-039 187 187 187 345 345 345 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 188 188 188 346 346 346 F F F 4 080-099 186 186 186 344 344 344 F F F 5 100-119 186 186 186 344 344 344 F F F 6 120-139 189 189 189 350 350 350 F F F 7 140-159 189 189 189 350 350 350 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 189 189 189 350 350 350 F F F 12 240-259 174 174 174 324 324 324 F F F 13 260-279 188 188 188 348 348 348 F F F 14 280-299 187 187 187 345 345 345 F F F 15 300-319 188 188 188 346 346 346 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 172 172 172 321 321 321 F F F 19 380-399 188 188 188 346 346 346 F F F 20 400-419 187 187 187 345 345 345 F F F 21 420-439 185 185 185 342 342 342 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 188 188 188 349 349 349 F F F 25 500-519 187 187 187 345 345 345 F F F 26 520-539 187 187 187 345 345 345 F F F 27 540-559 189 189 189 350 350 350 F F F 28 560-579 182 182 182 334 334 334 F F F 29 580-599 188 188 188 346 346 346 F F F Max gen: 189, Max flush: 350 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.550803419Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.596870262Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.597838854Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.597877167Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.597892294Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.597906692Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945} {"msg":"Crucible c5897f3e-7d21-4aba-9833-d51dacf4e344 has session id: 7ef4945c-1007-4e76-b4af-fb6a9517741d","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.598353274Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Upstairs opts: Upstairs UUID: c5897f3e-7d21-4aba-9833-d51dacf4e344, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.598399166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Crucible stats registered with UUID: c5897f3e-7d21-4aba-9833-d51dacf4e344","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.598424542Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.63021248Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945} {"msg":"c5897f3e-7d21-4aba-9833-d51dacf4e344 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630281538Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630309593Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630343537Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630369903Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630398028Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630422753Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630447819Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630473475Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"io task","client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630502319Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"io task","client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630519527Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"io task","client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56264)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630566279Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"io task","client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47438)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630587575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"io task","client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62128)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.630828263Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"io task","client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.631498856Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.631527511Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.631543288Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.631557856Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.631813102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.631854204Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.63233698Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.632365845Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.63239616Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633042137Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633066683Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633081441Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633173915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633196871Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633211068Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633223426Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633236794Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[0]R flush_numbers[0..12]: [347, 345, 330, 346, 344, 344, 350, 350, 341, 341, 347, 350]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633250651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[0]R generation[0..12]: [188, 187, 180, 188, 186, 186, 189, 189, 184, 184, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633265089Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633279166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[1]R flush_numbers[0..12]: [347, 345, 330, 346, 344, 344, 350, 350, 341, 341, 347, 350]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633293144Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[1]R generation[0..12]: [188, 187, 180, 188, 186, 186, 189, 189, 184, 184, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633306691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633320589Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[2]R flush_numbers[0..12]: [347, 345, 330, 346, 344, 344, 350, 350, 341, 341, 347, 350]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633334327Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[2]R generation[0..12]: [188, 187, 180, 188, 186, 186, 189, 189, 184, 184, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633348104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633361932Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633377869Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633390957Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Next flush: 351","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633403805Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633416632Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"":"downstairs","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.63343002Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633442048Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"0","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633472313Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"1","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.63348572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"client":"2","":"downstairs","session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633498858Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633511016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"c5897f3e-7d21-4aba-9833-d51dacf4e344 is now active with session: 7ef4945c-1007-4e76-b4af-fb6a9517741d","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633524094Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.633536102Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945,"session_id":"7ef4945c-1007-4e76-b4af-fb6a9517741d"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:11.63354832Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 413 len 16384 data: 15 15 18 16 02/30 Write block 364 len 4096 data: 22 03/30 Read block 595 len 8192 04/30 Read block 428 len 8192 05/30 Read block 250 len 24576 06/30 Read block 352 len 12288 07/30 Read block 259 len 8192 08/30 Read block 318 len 28672 09/30 Write block 361 len 24576 data: 19 22 21 23 18 20 10/30 Write block 94 len 16384 data: 13 12 14 13 11/30 Write block 378 len 28672 data: 12 13 12 17 15 14 13 12/30 Write block 232 len 40960 data: 12 14 16 15 14 14 13 14 15 14 13/30 Write block 312 len 12288 data: 15 10 9 14/30 Write block 61 len 16384 data: 9 14 15 14 15/30 Read block 21 len 16384 16/30 Write block 556 len 24576 data: 15 16 13 13 12 13 17/30 Write block 9 len 4096 data: 15 18/30 Write block 226 len 8192 data: 14 13 19/30 Read block 151 len 12288 20/30 Write block 491 len 40960 data: 14 11 9 12 13 12 15 16 16 13 21/30 Read block 242 len 12288 22/30 Write block 266 len 24576 data: 11 11 9 10 11 12 23/30 Read block 316 len 4096 24/30 Read block 50 len 12288 25/30 Write block 125 len 8192 data: 19 15 26/30 Read block 348 len 36864 27/30 Read block 98 len 16384 28/30 Write block 87 len 20480 data: 17 16 15 13 11 29/30 Read block 82 len 40960 30/30 Write block 403 len 12288 data: 12 17 18 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 4 Done Done Done false 2 Acked 1001 Write 1 Done Done Done false 3 Acked 1002 Read 2 Done Done Done false 4 Acked 1003 Read 2 Done Done Done false 5 Acked 1004 Read 6 Done Done Done false 6 Acked 1005 Read 3 Done Done Done false 7 Acked 1006 Read 2 Done Done Done false 8 Acked 1007 Read 7 Done Done Done false 9 Acked 1008 Write 6 Done Done Done false 10 Acked 1009 Write 4 Done Done Done false 11 Acked 1010 Write 7 Done Done Done false 12 Acked 1011 Write 10 Done Done Done false 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Write 4 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Write 6 Done Done Done false 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Read 3 Done Done Done false 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:11.654225011Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1945} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 347 347 347 T T T 1 020-039 187 187 187 345 345 345 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 188 188 188 346 346 346 T T T 4 080-099 186 186 186 344 344 344 T T T 5 100-119 186 186 186 344 344 344 F F F 6 120-139 189 189 189 350 350 350 T T T 7 140-159 189 189 189 350 350 350 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 189 189 189 350 350 350 T T T 12 240-259 174 174 174 324 324 324 T T T 13 260-279 188 188 188 348 348 348 T T T 14 280-299 187 187 187 345 345 345 F F F 15 300-319 188 188 188 346 346 346 T T T 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 172 172 172 321 321 321 T T T 19 380-399 188 188 188 346 346 346 T T T 20 400-419 187 187 187 345 345 345 T T T 21 420-439 185 185 185 342 342 342 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 188 188 188 349 349 349 T T T 25 500-519 187 187 187 345 345 345 T T T 26 520-539 187 187 187 345 345 345 F F F 27 540-559 189 189 189 350 350 350 T T T 28 560-579 182 182 182 334 334 334 T T T 29 580-599 188 188 188 346 346 346 F F F Max gen: 189, Max flush: 350 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 351 351 351 F F F 1 020-039 187 187 187 345 345 345 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 190 190 190 351 351 351 F F F 5 100-119 186 186 186 344 344 344 F F F 6 120-139 190 190 190 351 351 351 F F F 7 140-159 189 189 189 350 350 350 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 190 190 190 351 351 351 F F F 12 240-259 190 190 190 351 351 351 F F F 13 260-279 190 190 190 351 351 351 F F F 14 280-299 187 187 187 345 345 345 F F F 15 300-319 190 190 190 351 351 351 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 190 190 190 351 351 351 F F F 19 380-399 190 190 190 351 351 351 F F F 20 400-419 190 190 190 351 351 351 F F F 21 420-439 185 185 185 342 342 342 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 190 190 190 351 351 351 F F F 25 500-519 190 190 190 351 351 351 F F F 26 520-539 187 187 187 345 345 345 F F F 27 540-559 190 190 190 351 351 351 F F F 28 560-579 190 190 190 351 351 351 F F F 29 580-599 188 188 188 346 346 346 F F F Max gen: 190, Max flush: 351 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.132400778Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.178110184Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.17899343Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.179031083Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.17904932Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.179064448Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953} {"msg":"Crucible 77c1b292-6341-4d76-9627-d80508dd7aa9 has session id: 308b2e01-eb6d-4485-925b-de5ef7196936","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.179630829Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Upstairs opts: Upstairs UUID: 77c1b292-6341-4d76-9627-d80508dd7aa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.179679011Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Crucible stats registered with UUID: 77c1b292-6341-4d76-9627-d80508dd7aa9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.179705216Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210186192Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953} {"msg":"77c1b292-6341-4d76-9627-d80508dd7aa9 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210227905Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210245402Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210272857Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210290554Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210307791Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210324378Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210341765Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210357862Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"io task","client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210375129Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"io task","client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210398195Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"io task","client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_connection connected from Ok(127.0.0.1:39030)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210563116Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"io task","client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_connection connected from Ok(127.0.0.1:64807)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210608199Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"io task","client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_connection connected from Ok(127.0.0.1:46818)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.210639553Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"io task","client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.211450142Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.211478887Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.211494664Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.211508852Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.211580969Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.211620232Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.212198502Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.212230966Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.212349856Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.212936523Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.212974297Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213036496Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213061052Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213087027Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213110803Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213130789Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213152816Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[0]R flush_numbers[0..12]: [351, 345, 330, 351, 351, 344, 351, 350, 341, 341, 347, 351]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213176881Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[0]R generation[0..12]: [190, 187, 180, 190, 190, 186, 190, 189, 184, 184, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213202157Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213229552Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[1]R flush_numbers[0..12]: [351, 345, 330, 351, 351, 344, 351, 350, 341, 341, 347, 351]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213254108Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[1]R generation[0..12]: [190, 187, 180, 190, 190, 186, 190, 189, 184, 184, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213279314Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213307669Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[2]R flush_numbers[0..12]: [351, 345, 330, 351, 351, 344, 351, 350, 341, 341, 347, 351]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213365919Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[2]R generation[0..12]: [190, 187, 180, 190, 190, 186, 190, 189, 184, 184, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213398703Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213431917Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Max found gen is 191","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213461492Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Generation requested: 192 >= found:191","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213493206Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213521412Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213546797Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"":"downstairs","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213563614Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213575812Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"0","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.21358899Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"1","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213602038Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"client":"2","":"downstairs","session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213615225Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213627553Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"77c1b292-6341-4d76-9627-d80508dd7aa9 is now active with session: 308b2e01-eb6d-4485-925b-de5ef7196936","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213640391Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213652249Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953,"session_id":"308b2e01-eb6d-4485-925b-de5ef7196936"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:16.213664477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 392 len 8192 data: 10 9 02/30 Flush 03/30 Flush 04/30 Write block 104 len 12288 data: 13 14 18 05/30 Read block 44 len 28672 06/30 Flush 07/30 Write block 299 len 16384 data: 6 8 7 8 08/30 Write block 577 len 12288 data: 19 17 18 09/30 Write block 564 len 36864 data: 13 14 12 12 11 10 9 12 16 10/30 Flush 11/30 Flush 12/30 Write block 429 len 16384 data: 10 13 12 14 13/30 Write block 493 len 16384 data: 10 13 14 13 14/30 Write block 565 len 12288 data: 15 13 13 15/30 Write block 430 len 12288 data: 14 13 15 16/30 Read block 536 len 24576 17/30 Read block 217 len 4096 18/30 Write block 26 len 36864 data: 8 8 8 11 11 12 11 13 17 19/30 Write block 247 len 24576 data: 10 14 13 12 12 8 20/30 Read block 215 len 28672 21/30 Write block 364 len 8192 data: 24 19 22/30 Read block 209 len 12288 23/30 Flush 24/30 Flush 25/30 Write block 589 len 28672 data: 13 15 17 19 16 14 11 26/30 Read block 585 len 4096 27/30 Read block 225 len 20480 28/30 Write block 407 len 40960 data: 19 19 18 18 17 15 16 16 19 17 29/30 Read block 12 len 20480 30/30 Write block 592 len 32768 data: 20 17 15 12 9 6 6 5 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Sent Done false 26 Acked 1025 Read 1 Done Sent Done false 27 Acked 1026 Read 5 Done Sent Done false 28 Acked 1027 Write 10 Done Sent Done false 29 Acked 1028 Read 5 Done Sent Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 6 1 8 Done 5 0 5 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:16.266907581Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1953} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 351 351 351 F F F 1 020-039 192 192 192 357 357 357 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 190 190 190 351 351 351 F F F 5 100-119 192 192 192 354 354 354 F F F 6 120-139 190 190 190 351 351 351 F F F 7 140-159 189 189 189 350 350 350 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 190 190 190 351 351 351 F F F 12 240-259 192 192 192 357 357 357 F F F 13 260-279 190 190 190 351 351 351 F F F 14 280-299 192 192 192 355 355 355 F F F 15 300-319 192 192 192 355 355 355 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 192 192 192 357 357 357 F F F 19 380-399 192 192 192 352 352 352 F F F 20 400-419 190 190 190 351 351 351 T F T <--- 21 420-439 192 192 192 357 357 357 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 192 192 192 357 357 357 F F F 25 500-519 190 190 190 351 351 351 F F F 26 520-539 187 187 187 345 345 345 F F F 27 540-559 190 190 190 351 351 351 F F F 28 560-579 192 192 192 357 357 357 F F F 29 580-599 188 188 188 346 346 346 T T T Max gen: 192, Max flush: 357 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 351 351 351 F F F 1 020-039 192 192 192 357 357 357 F F F 2 040-059 180 180 180 330 330 330 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 190 190 190 351 351 351 F F F 5 100-119 192 192 192 354 354 354 F F F 6 120-139 190 190 190 351 351 351 F F F 7 140-159 189 189 189 350 350 350 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 190 190 190 351 351 351 F F F 12 240-259 192 192 192 357 357 357 F F F 13 260-279 190 190 190 351 351 351 F F F 14 280-299 192 192 192 355 355 355 F F F 15 300-319 192 192 192 355 355 355 F F F 16 320-339 177 177 177 328 328 328 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 192 192 192 357 357 357 F F F 19 380-399 192 192 192 352 352 352 F F F 20 400-419 193 193 193 358 358 358 F F F 21 420-439 192 192 192 357 357 357 F F F 22 440-459 183 183 183 336 336 336 F F F 23 460-479 181 181 181 331 331 331 F F F 24 480-499 192 192 192 357 357 357 F F F 25 500-519 190 190 190 351 351 351 F F F 26 520-539 187 187 187 345 345 345 F F F 27 540-559 190 190 190 351 351 351 F F F 28 560-579 192 192 192 357 357 357 F F F 29 580-599 193 193 193 358 358 358 F F F Max gen: 193, Max flush: 358 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.557322243Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:20.600366133Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.600893371Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.63296223Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.633815352Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.633853695Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.633868323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.63388268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961} {"msg":"Crucible f24bf3b2-1a44-4358-b294-3455b40d4d05 has session id: 1b1627b3-4643-4bb5-b3c5-ef0e5836ae48","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.634367526Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Upstairs opts: Upstairs UUID: f24bf3b2-1a44-4358-b294-3455b40d4d05, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.634417667Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Crucible stats registered with UUID: f24bf3b2-1a44-4358-b294-3455b40d4d05","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.634447822Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665523564Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961} {"msg":"f24bf3b2-1a44-4358-b294-3455b40d4d05 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665579924Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665599571Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665628676Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665646403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66566392Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665681057Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665697894Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665714401Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"io task","client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.665743706Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"io task","client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66577848Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"io task","client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_connection connected from Ok(127.0.0.1:46626)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66589268Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"io task","client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_connection connected from Ok(127.0.0.1:64077)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66594847Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"io task","client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_connection connected from Ok(127.0.0.1:43345)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.666168112Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"io task","client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.666951296Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.666991809Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.667021983Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.667046599Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.667061827Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.667089132Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.667648504Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.667842691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66795888Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668298931Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668326926Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668341254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668429878Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668454194Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668468382Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668523742Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668542849Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[0]R flush_numbers[0..12]: [351, 357, 330, 351, 351, 354, 351, 350, 341, 341, 347, 351]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668567054Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[0]R generation[0..12]: [190, 192, 180, 190, 190, 192, 190, 189, 184, 184, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668581572Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668595949Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[1]R flush_numbers[0..12]: [351, 357, 330, 351, 351, 354, 351, 350, 341, 341, 347, 351]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668610027Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[1]R generation[0..12]: [190, 192, 180, 190, 190, 192, 190, 189, 184, 184, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668623615Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668637272Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[2]R flush_numbers[0..12]: [351, 357, 330, 351, 351, 354, 351, 350, 341, 341, 347, 351]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66865071Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[2]R generation[0..12]: [190, 192, 180, 190, 190, 192, 190, 189, 184, 184, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668664228Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668677985Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668690693Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66870765Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Next flush: 359","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668720938Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668733596Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"":"downstairs","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T17:17:20.668746653Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668769069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"0","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668782177Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"1","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668795025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"client":"2","":"downstairs","session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668808132Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.66882017Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"f24bf3b2-1a44-4358-b294-3455b40d4d05 is now active with session: 1b1627b3-4643-4bb5-b3c5-ef0e5836ae48","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668832968Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668845016Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961,"session_id":"1b1627b3-4643-4bb5-b3c5-ef0e5836ae48"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:20.668857394Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 36864 data: 12 11 12 15 18 18 17 18 18 02/30 Write block 567 len 12288 data: 14 12 11 03/30 Read block 243 len 24576 04/30 Read block 54 len 8192 05/30 Flush 06/30 Write block 332 len 28672 data: 16 16 19 24 23 22 24 07/30 Write block 401 len 12288 data: 14 15 13 08/30 Write block 39 len 32768 data: 13 13 16 14 15 13 13 12 09/30 Flush 10/30 Write block 468 len 8192 data: 13 10 11/30 Write block 448 len 32768 data: 14 14 16 16 17 17 19 19 12/30 Write block 538 len 28672 data: 17 18 19 18 18 13 14 13/30 Write block 223 len 4096 data: 14 14/30 Read block 536 len 20480 15/30 Write block 381 len 36864 data: 18 16 15 14 15 15 13 14 14 16/30 Flush 17/30 Write block 98 len 20480 data: 13 14 14 14 17 18/30 Read block 275 len 40960 19/30 Write block 83 len 12288 data: 19 21 19 20/30 Read block 40 len 20480 21/30 Write block 496 len 28672 data: 14 16 17 17 14 16 16 22/30 Read block 195 len 28672 23/30 Read block 181 len 8192 24/30 Write block 279 len 4096 data: 10 25/30 Read block 44 len 28672 26/30 Write block 176 len 16384 data: 11 11 11 10 27/30 Write block 437 len 20480 data: 9 10 12 9 8 28/30 Write block 23 len 4096 data: 10 29/30 Write block 511 len 32768 data: 14 15 17 18 20 21 19 18 30/30 Write block 10 len 20480 data: 16 15 17 20 17 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 5 Done Done Sent false 28 Acked 1027 Write 1 Sent Done Sent false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 4 9 Done 11 12 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:20.691277972Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1961} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 351 351 351 T T T 1 020-039 194 194 194 360 360 360 T T T 2 040-059 194 194 194 360 360 360 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 190 190 190 351 351 351 T T T 5 100-119 192 192 192 354 354 354 T T T 6 120-139 190 190 190 351 351 351 F F F 7 140-159 194 194 194 359 359 359 F F F 8 160-179 194 194 194 359 359 359 T T T 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 194 194 194 361 361 361 F F F 12 240-259 192 192 192 357 357 357 F F F 13 260-279 190 190 190 351 351 351 T T T 14 280-299 192 192 192 355 355 355 F F F 15 300-319 192 192 192 355 355 355 F F F 16 320-339 194 194 194 360 360 360 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 192 192 192 357 357 357 F F F 19 380-399 194 194 194 361 361 361 F F F 20 400-419 194 194 194 360 360 360 F F F 21 420-439 192 192 192 357 357 357 T T T 22 440-459 194 194 194 361 361 361 T T T 23 460-479 194 194 194 361 361 361 F F F 24 480-499 192 192 192 357 357 357 T T T 25 500-519 190 190 190 351 351 351 T T T 26 520-539 194 194 194 361 361 361 F F F 27 540-559 194 194 194 361 361 361 F F F 28 560-579 194 194 194 359 359 359 F F F 29 580-599 193 193 193 358 358 358 F F F Max gen: 194, Max flush: 361 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 362 362 362 F F F 1 020-039 195 195 195 362 362 362 F F F 2 040-059 194 194 194 360 360 360 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 195 195 195 362 362 362 F F F 5 100-119 195 195 195 362 362 362 F F F 6 120-139 190 190 190 351 351 351 F F F 7 140-159 194 194 194 359 359 359 F F F 8 160-179 195 195 195 362 362 362 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 188 188 188 347 347 347 F F F 11 220-239 194 194 194 361 361 361 F F F 12 240-259 192 192 192 357 357 357 F F F 13 260-279 195 195 195 362 362 362 F F F 14 280-299 192 192 192 355 355 355 F F F 15 300-319 192 192 192 355 355 355 F F F 16 320-339 194 194 194 360 360 360 F F F 17 340-359 184 184 184 337 337 337 F F F 18 360-379 192 192 192 357 357 357 F F F 19 380-399 194 194 194 361 361 361 F F F 20 400-419 194 194 194 360 360 360 F F F 21 420-439 195 195 195 362 362 362 F F F 22 440-459 195 195 195 362 362 362 F F F 23 460-479 194 194 194 361 361 361 F F F 24 480-499 195 195 195 362 362 362 F F F 25 500-519 195 195 195 362 362 362 F F F 26 520-539 194 194 194 361 361 361 F F F 27 540-559 194 194 194 361 361 361 F F F 28 560-579 194 194 194 359 359 359 F F F 29 580-599 193 193 193 358 358 358 F F F Max gen: 195, Max flush: 362 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.082275281Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.12727838Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.128233963Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.128275026Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.128290173Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.128305091Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969} {"msg":"Crucible 8971032e-b767-4e83-9a7e-b148d4663225 has session id: 405f52fc-e4c9-47ea-bc57-4e2e37555375","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.128831259Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Upstairs opts: Upstairs UUID: 8971032e-b767-4e83-9a7e-b148d4663225, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.128874892Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Crucible stats registered with UUID: 8971032e-b767-4e83-9a7e-b148d4663225","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.128904906Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160724139Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969} {"msg":"8971032e-b767-4e83-9a7e-b148d4663225 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160783449Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160803056Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.16083403Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160852477Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160870164Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160886771Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160903468Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160920555Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"io task","client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160937462Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"io task","client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.160954349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"io task","client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_connection connected from Ok(127.0.0.1:53048)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.161088246Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"io task","client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_connection connected from Ok(127.0.0.1:50693)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.161132238Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"io task","client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_connection connected from Ok(127.0.0.1:51867)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.161427257Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"io task","client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162016104Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162057517Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162087522Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162151511Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162182915Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162198313Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162739679Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162766484Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.162873735Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163334335Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.16336567Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163392545Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163415251Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163435218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163449155Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.16347741Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163492128Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[0]R flush_numbers[0..12]: [362, 362, 360, 351, 362, 362, 351, 359, 362, 341, 347, 361]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163559006Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[0]R generation[0..12]: [195, 195, 194, 190, 195, 195, 190, 194, 195, 184, 188, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163579383Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.16359366Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[1]R flush_numbers[0..12]: [362, 362, 360, 351, 362, 362, 351, 359, 362, 341, 347, 361]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163607518Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[1]R generation[0..12]: [195, 195, 194, 190, 195, 195, 190, 194, 195, 184, 188, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163621025Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163634593Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[2]R flush_numbers[0..12]: [362, 362, 360, 351, 362, 362, 351, 359, 362, 341, 347, 361]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163648111Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[2]R generation[0..12]: [195, 195, 194, 190, 195, 195, 190, 194, 195, 184, 188, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163661478Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163675096Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163687764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163700651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Next flush: 363","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163713349Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163729237Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"":"downstairs","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163743854Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1"Wait for a query_work_queue command to finish before sending IO ,"pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.16376661Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"0","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163779728Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"1","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163792586Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"client":"2","":"downstairs","session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163805503Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163817651Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"8971032e-b767-4e83-9a7e-b148d4663225 is now active with session: 405f52fc-e4c9-47ea-bc57-4e2e37555375","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163845306Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163859144Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969,"session_id":"405f52fc-e4c9-47ea-bc57-4e2e37555375"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:25.163872121Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 34 len 40960 data: 18 12 11 12 13 14 14 17 15 16 02/30 Read block 319 len 36864 03/30 Read block 183 len 20480 04/30 Flush 05/30 Write block 342 len 12288 data: 19 16 15 06/30 Read block 89 len 12288 07/30 Read block 400 len 8192 08/30 Read block 361 len 40960 09/30 Write block 287 len 12288 data: 18 19 19 10/30 Flush 11/30 Write block 208 len 36864 data: 12 16 16 16 13 14 15 11 13 12/30 Read block 247 len 36864 13/30 Write block 237 len 40960 data: 15 14 15 16 15 13 13 11 12 12 14/30 Write block 436 len 24576 data: 9 10 11 13 10 9 15/30 Read block 548 len 4096 16/30 Write block 259 len 16384 data: 12 13 17 16 17/30 Read block 281 len 16384 18/30 Write block 427 len 24576 data: 10 11 11 15 14 16 19/30 Read block 441 len 36864 20/30 Write block 152 len 16384 data: 11 10 10 9 21/30 Read block 542 len 4096 22/30 Write block 456 len 20480 data: 23 21 17 14 15 23/30 Read block 210 len 12288 24/30 Flush 25/30 Write block 248 len 20480 data: 15 14 13 13 9 26/30 Read block 574 len 40960 27/30 Write block 422 len 16384 data: 18 15 14 14 28/30 Read block 499 len 40960 29/30 Read block 364 len 28672 30/30 Write block 147 len 24576 data: 11 9 10 11 12 12 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:25.192393218Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1969} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 362 362 362 F F F 1 020-039 196 196 196 363 363 363 F F F 2 040-059 196 196 196 363 363 363 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 195 195 195 362 362 362 F F F 5 100-119 195 195 195 362 362 362 F F F 6 120-139 190 190 190 351 351 351 F F F 7 140-159 196 196 196 365 365 365 T T T 8 160-179 195 195 195 362 362 362 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 196 196 196 365 365 365 F F F 11 220-239 196 196 196 365 365 365 F F F 12 240-259 196 196 196 365 365 365 T T T 13 260-279 196 196 196 365 365 365 F F F 14 280-299 196 196 196 364 364 364 F F F 15 300-319 192 192 192 355 355 355 F F F 16 320-339 194 194 194 360 360 360 F F F 17 340-359 196 196 196 364 364 364 F F F 18 360-379 192 192 192 357 357 357 F F F 19 380-399 194 194 194 361 361 361 F F F 20 400-419 194 194 194 360 360 360 F F F 21 420-439 196 196 196 365 365 365 T T T 22 440-459 196 196 196 365 365 365 F F F 23 460-479 196 196 196 365 365 365 F F F 24 480-499 195 195 195 362 362 362 F F F 25 500-519 195 195 195 362 362 362 F F F 26 520-539 194 194 194 361 361 361 F F F 27 540-559 194 194 194 361 361 361 F F F 28 560-579 194 194 194 359 359 359 F F F 29 580-599 193 193 193 358 358 358 F F F Max gen: 196, Max flush: 365 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 362 362 362 F F F 1 020-039 196 196 196 363 363 363 F F F 2 040-059 196 196 196 363 363 363 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 195 195 195 362 362 362 F F F 5 100-119 195 195 195 362 362 362 F F F 6 120-139 190 190 190 351 351 351 F F F 7 140-159 197 197 197 366 366 366 F F F 8 160-179 195 195 195 362 362 362 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 196 196 196 365 365 365 F F F 11 220-239 196 196 196 365 365 365 F F F 12 240-259 197 197 197 366 366 366 F F F 13 260-279 196 196 196 365 365 365 F F F 14 280-299 196 196 196 364 364 364 F F F 15 300-319 192 192 192 355 355 355 F F F 16 320-339 194 194 194 360 360 360 F F F 17 340-359 196 196 196 364 364 364 F F F 18 360-379 192 192 192 357 357 357 F F F 19 380-399 194 194 194 361 361 361 F F F 20 400-419 194 194 194 360 360 360 F F F 21 420-439 197 197 197 366 366 366 F F F 22 440-459 196 196 196 365 365 365 F F F 23 460-479 196 196 196 365 365 365 F F F 24 480-499 195 195 195 362 362 362 F F F 25 500-519 195 195 195 362 362 362 F F F 26 520-539 194 194 194 361 361 361 F F F 27 540-559 194 194 194 361 361 361 F F F 28 560-579 194 194 194 359 359 359 F F F 29 580-599 193 193 193 358 358 358 F F F Max gen: 197, Max flush: 366 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.48913212Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.533221847Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.534149856Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.534197228Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.534215034Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.534233081Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977} {"msg":"Crucible 940921c7-1943-417d-98b3-5153e06f2240 has session id: 32d16821-f9d7-4049-a032-fa1ab2013122","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.53464069Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Upstairs opts: Upstairs UUID: 940921c7-1943-417d-98b3-5153e06f2240, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.534695761Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Crucible stats registered with UUID: 940921c7-1943-417d-98b3-5153e06f2240","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.534727035Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.56630229Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977} {"msg":"940921c7-1943-417d-98b3-5153e06f2240 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.56636479Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566406123Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566456764Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566500166Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.5665331Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566563195Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.56659368Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566623445Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"io task","client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566643441Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"io task","client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566746623Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"io task","client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_connection connected from Ok(127.0.0.1:55581)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566780337Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"io task","client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_connection connected from Ok(127.0.0.1:35111)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.566812072Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"io task","client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_connection connected from Ok(127.0.0.1:64779)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.567194605Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"io task","client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.567752648Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.567785323Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.56780197Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.567815907Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.567899083Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.567944885Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.568516315Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.568664799Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.568700373Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569044813Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569070169Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569084317Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569276743Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569301029Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569315046Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569340322Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569354659Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[0]R flush_numbers[0..12]: [362, 363, 363, 351, 362, 362, 351, 366, 362, 341, 365, 365]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569440994Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[0]R generation[0..12]: [195, 196, 196, 190, 195, 195, 190, 197, 195, 184, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569463031Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569481217Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[1]R flush_numbers[0..12]: [362, 363, 363, 351, 362, 362, 351, 366, 362, 341, 365, 365]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569495575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[1]R generation[0..12]: [195, 196, 196, 190, 195, 195, 190, 197, 195, 184, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569509252Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.56952303Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[2]R flush_numbers[0..12]: [362, 363, 363, 351, 362, 362, 351, 366, 362, 341, 365, 365]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569536988Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[2]R generation[0..12]: [195, 196, 196, 190, 195, 195, 190, 197, 195, 184, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569550535Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569564473Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569577181Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569591318Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Next flush: 367","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569604286Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569617254Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"":"downstairs","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569665575Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569702529Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"0","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569730954Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"1","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-09-27T17:17:29.569761768Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"client":"2","":"downstairs","session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569804691Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569826987Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"940921c7-1943-417d-98b3-5153e06f2240 is now active with session: 32d16821-f9d7-4049-a032-fa1ab2013122","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569847784Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.56986738Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977,"session_id":"32d16821-f9d7-4049-a032-fa1ab2013122"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:17:29.569888386Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 36864 data: 19 18 16 15 9 10 12 13 17 02/30 Read block 241 len 28672 03/30 Read block 420 len 20480 04/30 Write block 136 len 32768 data: 10 12 16 16 14 12 12 10 05/30 Read block 446 len 12288 06/30 Read block 163 len 4096 07/30 Read block 305 len 40960 08/30 Write block 255 len 28672 data: 9 11 10 10 13 14 18 09/30 Write block 204 len 24576 data: 16 10 11 13 14 18 10/30 Write block 236 len 28672 data: 15 16 15 16 17 16 14 11/30 Write block 128 len 24576 data: 11 9 10 9 12 10 12/30 Flush 13/30 Write block 22 len 40960 data: 11 11 11 11 9 9 9 12 12 13 14/30 Read block 501 len 24576 15/30 Read block 48 len 4096 16/30 Write block 283 len 24576 data: 14 13 17 17 19 20 17/30 Read block 98 len 12288 18/30 Read block 21 len 24576 19/30 Read block 213 len 28672 20/30 Write block 14 len 24576 data: 18 19 19 19 20 20 21/30 Write block 362 len 8192 data: 23 22 22/30 Read block 66 len 40960 23/30 Read block 407 len 36864 24/30 Read block 391 len 40960 25/30 Write block 5 len 20480 data: 14 16 16 14 16 26/30 Read block 250 len 16384 27/30 Write block 556 len 28672 data: 16 17 14 14 13 14 15 28/30 Read block 441 len 36864 29/30 Write block 446 len 8192 data: 16 17 30/30 Read block 195 len 20480 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 10 Done Done Done false 14 Acked 1013 Read 6 Done Done Done false 15 Acked 1014 Read 1 Done Done Done false 16 Acked 1015 Write 6 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Read 6 Done Done Done false 19 Acked 1018 Read 7 Done Done Done false 20 Acked 1019 Write 6 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 2 Done Sent Sent false 30 Acked 1029 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 18 16 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:17:29.59383365Z","hostname":"w-01J8T79QZR3J4W60V8JCSQYXQ1","pid":1977} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 362 362 362 T T T 1 020-039 196 196 196 363 363 363 T T T 2 040-059 196 196 196 363 363 363 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 195 195 195 362 362 362 F F F 5 100-119 195 195 195 362 362 362 F F F 6 120-139 198 198 198 367 367 367 F F F 7 140-159 198 198 198 367 367 367 F F F 8 160-179 195 195 195 362 362 362 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 198 198 198 367 367 367 F F F 11 220-239 198 198 198 367 367 367 F F F 12 240-259 198 198 198 367 367 367 F F F 13 260-279 198 198 198 367 367 367 F F F 14 280-299 196 196 196 364 364 364 T T T 15 300-319 192 192 192 355 355 355 F F F 16 320-339 194 194 194 360 360 360 F F F 17 340-359 196 196 196 364 364 364 F F F 18 360-379 192 192 192 357 357 357 T T T 19 380-399 194 194 194 361 361 361 F F F 20 400-419 194 194 194 360 360 360 F F F 21 420-439 197 197 197 366 366 366 F F F 22 440-459 196 196 196 365 365 365 T T T 23 460-479 196 196 196 365 365 365 F F F 24 480-499 195 195 195 362 362 362 F F F 25 500-519 195 195 195 362 362 362 F F F 26 520-539 194 194 194 361 361 361 F F F 27 540-559 194 194 194 361 361 361 T T T 28 560-579 194 194 194 359 359 359 T T T 29 580-599 193 193 193 358 358 358 F F F Max gen: 198, Max flush: 367 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 368 368 368 F F F 1 020-039 199 199 199 368 368 368 F F F 2 040-059 196 196 196 363 363 363 F F F 3 060-079 190 190 190 351 351 351 F F F 4 080-099 195 195 195 362 362 362 F F F 5 100-119 195 195 195 362 362 362 F F F 6 120-139 198 198 198 367 367 367 F F F 7 140-159 198 198 198 367 367 367 F F F 8 160-179 195 195 195 362 362 362 F F F 9 180-199 184 184 184 341 341 341 F F F 10 200-219 198 198 198 367 367 367 F F F 11 220-239 198 198 198 367 367 367 F F F 12 240-259 198 198 198 367 367 367 F F F 13 260-279 198 198 198 367 367 367 F F F 14 280-299 199 199 199 368 368 368 F F F 15 300-319 192 192 192 355 355 355 F F F 16 320-339 194 194 194 360 360 360 F F F 17 340-359 196 196 196 364 364 364 F F F 18 360-379 199 199 199 368 368 368 F F F 19 380-399 194 194 194 361 361 361 F F F 20 400-419 194 194 194 360 360 360 F F F 21 420-439 197 197 197 366 366 366 F F F 22 440-459 199 199 199 368 368 368 F F F 23 460-479 196 196 196 365 365 365 F F F 24 480-499 195 195 195 362 362 362 F F F 25 500-519 195 195 195 362 362 362 F F F 26 520-539 194 194 194 361 361 361 F F F 27 540-559 199 199 199 368 368 368 F F F 28 560-579 199 199 199 368 368 368 F F F 29 580-599 193 193 193 358 358 358 F F F Max gen: 199, Max flush: 368 7:37 Test duration Test completed real 7:37.700864294 user 57.696288354 sys 20.585939190 trap 0.071994396 tflt 0.035457679 dflt 0.078752832 kflt 0.007089007 lock 12:13:25.032192989 slp 37:08.101277801 lat 5.869289442 stop 0.062514125