Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.31527543 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.052309055 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.053186122 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1183 1184 1185 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.799294727Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.872908998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.874697741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.874818175Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.874844311Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.87491971Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187} {"msg":"Crucible 0ff922dc-1bd8-4cde-aa5f-9f0d86d0997e has session id: 4fbae1c0-0b14-4f65-9b6a-bc496a1103e1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.875750016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Upstairs opts: Upstairs UUID: 0ff922dc-1bd8-4cde-aa5f-9f0d86d0997e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.87586206Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Crucible stats registered with UUID: 0ff922dc-1bd8-4cde-aa5f-9f0d86d0997e","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.875883847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.916969122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187} {"msg":"0ff922dc-1bd8-4cde-aa5f-9f0d86d0997e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917022605Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917041832Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917180283Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917207009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917224727Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917252763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917281489Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917304276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917326763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.91734902Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42212)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917422959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47635)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917539383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49035)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.917569269Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.919396287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.919425983Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.919441411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.919455409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.919473126Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.919503022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.920622497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.920786415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.92082246Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921732804Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921768509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921791076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921811143Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.92183241Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921853667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921872565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921893112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921914839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921936256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.921958293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922009996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922033953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922056029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922077167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922098153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922119921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922140878Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922161965Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922181762Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922201979Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"":"downstairs","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922223716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922243604Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"0","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922264181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"1","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922283778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"client":"2","":"downstairs","session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922304225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922325852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"0ff922dc-1bd8-4cde-aa5f-9f0d86d0997e is now active with session: 4fbae1c0-0b14-4f65-9b6a-bc496a1103e1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922345359Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922363327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187Wait for a query_work_queue command to finish before sending IO ,"session_id":"4fbae1c0-0b14-4f65-9b6a-bc496a1103e1"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:17.922385464Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 5 Acked 1004 Read 88 Done Done Done false 6 Acked 1005 Read 256 Done Done Done false 7 Acked 1006 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 7 6 5 4 3 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 8 7 6 5 4 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:25.647589491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1187} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.680576314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:25.723692408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.724047299Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.755919017Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.756484078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.756523793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.756538211Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.756552709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189} {"msg":"Crucible 7f5f561c-bb50-4fcf-9bd5-bebcf539aa60 has session id: 905469a0-b062-44df-a4ce-32050c417417","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.756939346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Upstairs opts: Upstairs UUID: 7f5f561c-bb50-4fcf-9bd5-bebcf539aa60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.756997388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Crucible stats registered with UUID: 7f5f561c-bb50-4fcf-9bd5-bebcf539aa60","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.757024844Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.787759038Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189} {"msg":"7f5f561c-bb50-4fcf-9bd5-bebcf539aa60 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.787793603Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.78781199Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.787839777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.787857714Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.787874592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.78789089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.787963169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788014073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788048728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788082693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_connection connected from Ok(127.0.0.1:64989)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788116459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_connection connected from Ok(127.0.0.1:35918)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788147354Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_connection connected from Ok(127.0.0.1:53901)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788274257Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788859416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.78889929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788931596Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788962162Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.788989358Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.789011295Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.789509856Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.78969433Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.789723577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790030814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790067369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790091815Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790293808Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790333302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790361398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790390884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790416731Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790476602Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790502679Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790518047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790531795Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790545133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790559331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790572889Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790590667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790605795Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790618913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790632031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:50:25.790644769Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790660737Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"":"downstairs","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790674005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790685933Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"0","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790699192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"1","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.79071208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"client":"2","":"downstairs","session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790725328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790737456Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"7f5f561c-bb50-4fcf-9bd5-bebcf539aa60 is now active with session: 905469a0-b062-44df-a4ce-32050c417417","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790750364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790762153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189,"session_id":"905469a0-b062-44df-a4ce-32050c417417"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:25.790774371Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 260 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Write block 452 len 16384 data: 2 2 2 2 03/30 Read block 342 len 12288 04/30 Read block 490 len 8192 05/30 Read block 518 len 20480 06/30 Write block 346 len 12288 data: 2 2 2 07/30 Read block 47 len 4096 08/30 Read block 338 len 16384 09/30 Write block 69 len 28672 data: 2 2 2 2 2 2 2 10/30 Write block 591 len 8192 data: 2 2 11/30 Write block 119 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Flush 13/30 Write block 310 len 20480 data: 2 2 2 2 2 14/30 Write block 13 len 16384 data: 2 2 2 2 15/30 Write block 541 len 32768 data: 2 2 2 2 2 2 2 2 16/30 Write block 337 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Write block 85 len 28672 data: 2 2 2 2 2 2 2 18/30 Read block 318 len 20480 19/30 Read block 172 len 40960 20/30 Read block 288 len 12288 21/30 Read block 165 len 12288 22/30 Write block 129 len 24576 data: 2 2 2 2 2 2 23/30 Read block 37 len 36864 24/30 Flush 25/30 Flush 26/30 Write block 538 len 16384 data: 2 2 2 3 27/30 Read block 259 len 40960 28/30 Write block 540 len 8192 data: 3 4 29/30 Read block 290 len 4096 30/30 Read block 531 len 24576 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:25.993869037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1189} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 2 2 2 3 3 3 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 T T T 27 540-559 2 2 2 3 3 3 T T T 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 2, Max flush: 3 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 2 2 2 3 3 3 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 3 3 3 4 4 4 F F F 27 540-559 3 3 3 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.320146652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:30.36590762Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.366353029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.398510697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.399166936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.399202921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.399223209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.399238256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197} {"msg":"Crucible f15c5d31-ba4e-4250-9683-4fc9453eadb4 has session id: 21ea9d0a-5186-4c98-bef2-082580af5762","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.399734948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Upstairs opts: Upstairs UUID: f15c5d31-ba4e-4250-9683-4fc9453eadb4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.399786191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Crucible stats registered with UUID: f15c5d31-ba4e-4250-9683-4fc9453eadb4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.399819036Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432185975Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197} {"msg":"f15c5d31-ba4e-4250-9683-4fc9453eadb4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43222495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432243207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432283622Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432317227Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432358512Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432396546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432432261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432468366Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432508561Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432543606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_connection connected from Ok(127.0.0.1:59969)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432565113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_connection connected from Ok(127.0.0.1:65415)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432598288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_connection connected from Ok(127.0.0.1:34444)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.432630194Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.433496964Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.433536459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.433567114Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43359559Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.433623507Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.433648993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434151594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43417903Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434419767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43468319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434711936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434732113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43475782Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434772538Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434786176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434895601Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434914139Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[0]R flush_numbers[0..12]: [3, 1, 1, 2, 3, 2, 3, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434939515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[0]R generation[0..12]: [2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434953793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434967861Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[1]R flush_numbers[0..12]: [3, 1, 1, 2, 3, 2, 3, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434981529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[1]R generation[0..12]: [2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.434995097Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435009115Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[2]R flush_numbers[0..12]: [3, 1, 1, 2, 3, 2, 3, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435022553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[2]R generation[0..12]: [2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435035822Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43504963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435066517Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435080065Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435092994Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435106002Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"":"downstairs","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43511921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","Wait for a query_work_queue command to finish before sending IO pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.43519098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"0","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435206958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"1","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435220266Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"client":"2","":"downstairs","session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435233684Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435245933Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"f15c5d31-ba4e-4250-9683-4fc9453eadb4 is now active with session: 21ea9d0a-5186-4c98-bef2-082580af5762","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435258741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435270699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197,"session_id":"21ea9d0a-5186-4c98-bef2-082580af5762"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:30.435282937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 33 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 540 len 36864 data: 4 5 3 3 3 3 3 3 3 04/30 Read block 355 len 24576 05/30 Read block 69 len 28672 06/30 Read block 89 len 24576 07/30 Read block 358 len 12288 08/30 Flush 09/30 Write block 345 len 36864 data: 2 3 3 3 2 2 2 2 2 10/30 Read block 185 len 32768 11/30 Flush 12/30 Write block 51 len 4096 data: 2 13/30 Write block 111 len 12288 data: 2 2 2 14/30 Read block 578 len 8192 15/30 Write block 376 len 24576 data: 2 2 2 2 2 2 16/30 Read block 167 len 12288 17/30 Read block 170 len 28672 18/30 Write block 530 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Read block 503 len 8192 20/30 Read block 419 len 40960 21/30 Write block 45 len 32768 data: 2 2 2 2 2 2 3 2 22/30 Write block 104 len 20480 data: 2 2 2 2 2 23/30 Write block 63 len 36864 data: 2 2 2 2 2 2 3 3 3 24/30 Read block 263 len 12288 25/30 Read block 287 len 16384 26/30 Read block 320 len 20480 27/30 Read block 474 len 20480 28/30 Read block 122 len 16384 29/30 Read block 138 len 16384 30/30 Write block 290 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 12 Acked 1011 Write 1 Done Done Done false 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Read 2 Done Done Done false 15 Acked 1014 Write 6 Done Done Done false 16 Acked 1015 Read 3 Done Done Done false 17 Acked 1016 Read 7 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:30.461772858Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1197} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 4 4 4 5 5 5 T T T 3 060-079 2 2 2 2 2 2 T T T 4 080-099 2 2 2 3 3 3 F F F 5 100-119 2 2 2 2 2 2 T T T 6 120-139 2 2 2 3 3 3 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 3 3 3 4 4 4 T T T 27 540-559 4 4 4 6 6 6 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 4, Max flush: 7 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 5 5 5 8 8 8 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 8 8 8 F F F 27 540-559 4 4 4 6 6 6 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 5, Max flush: 8 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.831542424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.877287855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.877943624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.877980379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.877995497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.878009915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205} {"msg":"Crucible e239b2a5-1608-4b3f-99c1-970562d1f871 has session id: 2f36a0b0-9e00-47c4-8f47-26260212a90a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.878432857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Upstairs opts: Upstairs UUID: e239b2a5-1608-4b3f-99c1-970562d1f871, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.878471961Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Crucible stats registered with UUID: e239b2a5-1608-4b3f-99c1-970562d1f871","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.878495628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.90975849Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205} {"msg":"e239b2a5-1608-4b3f-99c1-970562d1f871 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909789735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909808263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909835399Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909852877Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909869854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909886552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909914958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909946824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.909980129Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910013904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45270)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910057478Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56838)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910197059Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33445)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910237064Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910812284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910850889Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910880675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.91090976Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910938767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.910968112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.91149737Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.911575959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.911682734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912125133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912154009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912169466Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912198602Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.91221664Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912230598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912254805Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912268643Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[0]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 8, 3, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912330844Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[0]R generation[0..12]: [2, 4, 5, 5, 2, 5, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912350861Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.91236529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[1]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 8, 3, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912378748Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[1]R generation[0..12]: [2, 4, 5, 5, 2, 5, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912392306Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912406144Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[2]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 8, 3, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912419562Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[2]R generation[0..12]: [2, 4, 5, 5, 2, 5, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.91243328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912447208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912460217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912473415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912489952Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"Wait for a query_work_queue command to finish before sending IO msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912509759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"":"downstairs","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912523098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912535146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"0","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912548204Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"1","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912561183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"client":"2","":"downstairs","session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912574241Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912586539Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"e239b2a5-1608-4b3f-99c1-970562d1f871 is now active with session: 2f36a0b0-9e00-47c4-8f47-26260212a90a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912599297Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912611085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205,"session_id":"2f36a0b0-9e00-47c4-8f47-26260212a90a"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:34.912623254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 169 len 8192 data: 2 2 03/30 Read block 269 len 16384 04/30 Write block 559 len 16384 data: 2 2 2 2 05/30 Write block 115 len 32768 data: 2 2 2 2 3 3 3 3 06/30 Write block 402 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Flush 08/30 Write block 546 len 12288 data: 4 4 4 09/30 Write block 130 len 4096 data: 3 10/30 Write block 497 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Flush 12/30 Write block 130 len 20480 data: 4 3 3 3 3 13/30 Read block 141 len 20480 14/30 Flush 15/30 Write block 461 len 24576 data: 2 2 2 2 2 2 16/30 Read block 190 len 20480 17/30 Read block 194 len 16384 18/30 Read block 582 len 20480 19/30 Read block 373 len 40960 20/30 Flush 21/30 Write block 470 len 28672 data: 2 2 2 2 2 2 2 22/30 Write block 514 len 8192 data: 2 2 23/30 Read block 481 len 12288 24/30 Read block 517 len 32768 25/30 Read block 584 len 24576 26/30 Read block 565 len 20480 27/30 Read block 499 len 24576 28/30 Read block 425 len 40960 29/30 Write block 404 len 8192 data: 3 3 30/30 Read block 561 len 20480 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Write 2 Done Sent Sent false 30 Acked 1029 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 10 8 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:34.938725168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1205} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 6 6 6 12 12 12 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 5 5 5 8 8 8 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 F F F 20 400-419 6 6 6 10 10 10 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 13 13 13 T T T 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 T T T 26 520-539 5 5 5 8 8 8 F F F 27 540-559 6 6 6 11 11 11 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 6, Max flush: 13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 6 6 6 12 12 12 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 5 5 5 8 8 8 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 F F F 20 400-419 7 7 7 14 14 14 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 7 7 7 14 14 14 F F F 26 520-539 5 5 5 8 8 8 F F F 27 540-559 6 6 6 11 11 11 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 7, Max flush: 14 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.224152535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.271036449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.271909198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.271946073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.271960411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.271974609Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213} {"msg":"Crucible 8baad4bf-7397-4587-9ab4-a0a247208969 has session id: e92694a1-aa28-48ac-8176-eaced3785fde","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.272396771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Upstairs opts: Upstairs UUID: 8baad4bf-7397-4587-9ab4-a0a247208969, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.272434536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Crucible stats registered with UUID: 8baad4bf-7397-4587-9ab4-a0a247208969","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.272459393Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303811902Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213} {"msg":"8baad4bf-7397-4587-9ab4-a0a247208969 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303859905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303878842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303906838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303924196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303941144Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303957791Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.303993387Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.304012634Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.304030792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.304047999Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_connection connected from Ok(127.0.0.1:53450)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.304245562Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_connection connected from Ok(127.0.0.1:56355)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.304297425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_connection connected from Ok(127.0.0.1:58574)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.304421927Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305082096Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305111182Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305130269Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305144847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305158356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305182362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305731486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.30577538Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.305857819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306238887Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306265433Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306280121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306414742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306442388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306457926Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306470724Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306484362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[0]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 10, 12, 1, 10, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.3065032Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[0]R generation[0..12]: [2, 4, 5, 5, 2, 6, 6, 1, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306517588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306531676Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[1]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 10, 12, 1, 10, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306545184Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[1]R generation[0..12]: [2, 4, 5, 5, 2, 6, 6, 1, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306558462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.30657236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[2]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 10, 12, 1, 10, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306585699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[2]R generation[0..12]: [2, 4, 5, 5, 2, 6, 6, 1, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306598907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306612695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306625703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306638861Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.30665159Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306664328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"":"downstairs","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306677316Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306689424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"0","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306715231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"1","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306728729Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"client":"2","":"downstairs","session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306741837Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306754215Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"8baad4bf-7397-4587-9ab4-a0a247208969 is now active with session: e92694a1-aa28-48ac-8176-eaced3785fde","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306767143Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.306779002Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213,"session_id":"e92694a1-aa28-48ac-8176-eaced3785fde"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:39.30679124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 107 len 24576 data: 3 3 2 2 3 3 02/30 Write block 153 len 40960 data: 2 2 2 2 2 2 2 2 2 2 03/30 Write block 114 len 12288 data: 2 3 3 04/30 Write block 556 len 24576 data: 2 2 2 3 3 3 05/30 Read block 93 len 32768 06/30 Read block 297 len 32768 07/30 Write block 564 len 28672 data: 2 2 2 2 2 2 2 08/30 Write block 151 len 8192 data: 2 2 09/30 Read block 266 len 36864 10/30 Read block 52 len 40960 11/30 Write block 258 len 20480 data: 2 2 3 3 3 12/30 Read block 49 len 40960 13/30 Read block 270 len 32768 14/30 Read block 156 len 8192 15/30 Read block 496 len 32768 16/30 Write block 291 len 12288 data: 3 3 3 17/30 Write block 571 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Write block 528 len 12288 data: 2 2 3 19/30 Read block 308 len 4096 20/30 Read block 172 len 24576 21/30 Read block 586 len 20480 22/30 Write block 218 len 20480 data: 2 2 2 2 2 23/30 Read block 231 len 24576 24/30 Write block 139 len 36864 data: 2 2 2 2 2 2 2 2 2 25/30 Write block 549 len 4096 data: 2 26/30 Write block 497 len 28672 data: 3 3 3 3 3 3 3 27/30 Read block 361 len 8192 28/30 Read block 458 len 4096 29/30 Read block 199 len 8192 30/30 Read block 19 len 8192 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Write 10 Done Done Done false 3 Acked 1002 Write 3 Done Done Done false 4 Acked 1003 Write 6 Done Done Done false 5 Acked 1004 Read 8 Done Done Done false 6 Acked 1005 Read 8 Done Done Done false 7 Acked 1006 Write 7 Done Done Done false 8 Acked 1007 Write 2 Done Done Done false 9 Acked 1008 Read 9 Done Done Done false 10 Acked 1009 Read 10 Done Done Done false 11 Acked 1010 Write 5 Done Done Done false 12 Acked 1011 Read 10 Done Done Done false 13 Acked 1012 Read 8 Done Done Done false 14 Acked 1013 Read 2 Done Done Done false 15 Acked 1014 Read 8 Done Done Done false 16 Acked 1015 Write 3 Done Done Done false 17 Acked 1016 Write 8 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Read 1 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:39.325632168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1213} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 6 6 6 10 10 10 T T T 6 120-139 6 6 6 12 12 12 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 6 6 6 10 10 10 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 2 2 2 2 2 2 T T T 14 280-299 5 5 5 8 8 8 T T T 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 F F F 20 400-419 7 7 7 14 14 14 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 6 6 6 11 11 11 T T T 25 500-519 7 7 7 14 14 14 T T T 26 520-539 5 5 5 8 8 8 T T T 27 540-559 6 6 6 11 11 11 T T T 28 560-579 6 6 6 10 10 10 T T T 29 580-599 2 2 2 2 2 2 F F F Max gen: 7, Max flush: 14 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 8 8 8 15 15 15 F F F 6 120-139 8 8 8 15 15 15 F F F 7 140-159 8 8 8 15 15 15 F F F 8 160-179 8 8 8 15 15 15 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 8 8 8 15 15 15 F F F 13 260-279 8 8 8 15 15 15 F F F 14 280-299 8 8 8 15 15 15 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 F F F 20 400-419 7 7 7 14 14 14 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 8 8 8 15 15 15 F F F 25 500-519 8 8 8 15 15 15 F F F 26 520-539 8 8 8 15 15 15 F F F 27 540-559 8 8 8 15 15 15 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 8, Max flush: 15 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.759111364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.803966307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.804641344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.804685998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.804703536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.804721573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221} {"msg":"Crucible 6159f1cc-05b8-4606-830e-9d4d0f9ecbbd has session id: 7a7088e1-733b-4c31-b6d2-453ab460f7d8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.805168971Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Upstairs opts: Upstairs UUID: 6159f1cc-05b8-4606-830e-9d4d0f9ecbbd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.805202847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Crucible stats registered with UUID: 6159f1cc-05b8-4606-830e-9d4d0f9ecbbd","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.805223484Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836322558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221} {"msg":"6159f1cc-05b8-4606-830e-9d4d0f9ecbbd active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836368562Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836401747Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.83644879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836483435Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836515771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836548886Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836584162Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836615597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836647763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836681918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37334)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836715274Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:44719)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836746129Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:60624)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.836779525Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.837453281Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.837485687Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.837501755Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.837515743Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.837529481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.837543009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838063817Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838123349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838336439Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838680682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838719477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838751342Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838778318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838792556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838805885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838891213Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.8389133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[0]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 15, 15, 15, 15, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838942826Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[0]R generation[0..12]: [2, 4, 5, 5, 2, 8, 8, 8, 8, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.838957094Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.83898455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[1]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 15, 15, 15, 15, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839001548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[1]R generation[0..12]: [2, 4, 5, 5, 2, 8, 8, 8, 8, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839015386Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839029314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[2]R flush_numbers[0..12]: [3, 5, 8, 8, 3, 15, 15, 15, 15, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839042982Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[2]R generation[0..12]: [2, 4, 5, 5, 2, 8, 8, 8, 8, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.83905661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839070588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Max found gen is 9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839083696Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Generation requested: 10 >= found:9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839096925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839109953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839122851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"":"downstairs","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:50:43.839136019Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839154427Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"0","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839167765Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"1","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839180963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"client":"2","":"downstairs","session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839194061Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839206259Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"6159f1cc-05b8-4606-830e-9d4d0f9ecbbd is now active with session: 7a7088e1-733b-4c31-b6d2-453ab460f7d8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839219058Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839231036Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221,"session_id":"7a7088e1-733b-4c31-b6d2-453ab460f7d8"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:43.839244504Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 418 len 20480 data: 2 2 2 2 2 02/30 Write block 454 len 24576 data: 3 3 2 2 2 2 03/30 Read block 129 len 12288 04/30 Write block 522 len 20480 data: 2 2 2 2 2 05/30 Write block 325 len 12288 data: 2 2 2 06/30 Write block 519 len 12288 data: 2 2 2 07/30 Write block 251 len 4096 data: 2 08/30 Read block 581 len 4096 09/30 Flush 10/30 Write block 403 len 36864 data: 3 4 4 3 3 3 3 2 2 11/30 Read block 350 len 36864 12/30 Write block 417 len 36864 data: 2 3 3 3 3 3 2 2 2 13/30 Write block 262 len 4096 data: 4 14/30 Write block 511 len 28672 data: 2 2 2 3 3 2 2 15/30 Read block 280 len 28672 16/30 Write block 92 len 20480 data: 2 2 2 2 2 17/30 Write block 138 len 12288 data: 2 3 3 18/30 Write block 167 len 4096 data: 2 19/30 Read block 194 len 32768 20/30 Write block 502 len 20480 data: 4 4 3 2 2 21/30 Write block 566 len 32768 data: 3 3 3 3 3 3 3 3 22/30 Write block 499 len 36864 data: 4 4 4 5 5 4 3 3 2 23/30 Write block 131 len 32768 data: 4 4 4 4 2 2 2 3 24/30 Read block 345 len 32768 25/30 Write block 346 len 32768 data: 4 4 4 3 3 3 3 3 26/30 Read block 76 len 20480 27/30 Read block 573 len 28672 28/30 Read block 150 len 16384 29/30 Write block 271 len 28672 data: 2 2 2 2 2 2 2 30/30 Read block 239 len 24576 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 10 Acked 1009 Write 9 Done Done Done false 11 Acked 1010 Read 9 Done Done Done false 12 Acked 1011 Write 9 Done Done Done false 13 Acked 1012 Write 1 Done Done Done false 14 Acked 1013 Write 7 Done Done Done false 15 Acked 1014 Read 7 Done Done Done false 16 Acked 1015 Write 5 Done Done Done false 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 7 Done Done Sent false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 21 20 19 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:43.86063086Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1221} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 2 2 2 3 3 3 T T T 5 100-119 8 8 8 15 15 15 F F F 6 120-139 8 8 8 15 15 15 T T T 7 140-159 8 8 8 15 15 15 T T T 8 160-179 8 8 8 15 15 15 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 8 8 8 15 15 15 T T T 14 280-299 8 8 8 15 15 15 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 10 10 10 16 16 16 F F F 17 340-359 4 4 4 7 7 7 T T T 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 F F F 20 400-419 10 10 10 16 16 16 T T T 21 420-439 10 10 10 16 16 16 T T T 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 8 8 8 15 15 15 T T T 25 500-519 10 10 10 16 16 16 T T T 26 520-539 10 10 10 16 16 16 F F F 27 540-559 8 8 8 15 15 15 F F F 28 560-579 8 8 8 15 15 15 T T T 29 580-599 2 2 2 2 2 2 F F F Max gen: 10, Max flush: 16 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 11 11 11 17 17 17 F F F 5 100-119 8 8 8 15 15 15 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 11 11 11 17 17 17 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 11 11 11 17 17 17 F F F 14 280-299 8 8 8 15 15 15 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 10 10 10 16 16 16 F F F 17 340-359 11 11 11 17 17 17 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 F F F 20 400-419 11 11 11 17 17 17 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 11 11 11 17 17 17 F F F 25 500-519 11 11 11 17 17 17 F F F 26 520-539 10 10 10 16 16 16 F F F 27 540-559 8 8 8 15 15 15 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 11, Max flush: 17 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.236856154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.28264208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.283724531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.283768524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.283786742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.283806229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229} {"msg":"Crucible b673c8f7-ea6a-4186-8b53-00fbf5045671 has session id: dd5118e7-03a1-43a4-bf56-14c52db18b60","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.284264516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Upstairs opts: Upstairs UUID: b673c8f7-ea6a-4186-8b53-00fbf5045671, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.284338576Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Crucible stats registered with UUID: b673c8f7-ea6a-4186-8b53-00fbf5045671","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.284369681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317091141Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229} {"msg":"b673c8f7-ea6a-4186-8b53-00fbf5045671 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317142834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317164141Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317200546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317237311Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317271737Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317302842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317334878Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317366383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.31739404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317421986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_connection connected from Ok(127.0.0.1:43000)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317450692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_connection connected from Ok(127.0.0.1:49337)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.31760896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_connection connected from Ok(127.0.0.1:41663)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.317732973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.31847788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.318523633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.31854569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.318563378Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.318713577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.31876796Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319276949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319304996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319320343Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319899083Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.31992597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319940728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319953256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319966694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319979802Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.319991731Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320005229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[0]R flush_numbers[0..12]: [3, 5, 8, 8, 17, 15, 17, 17, 17, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320023416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[0]R generation[0..12]: [2, 4, 5, 5, 11, 8, 11, 11, 11, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320037794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320052102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[1]R flush_numbers[0..12]: [3, 5, 8, 8, 17, 15, 17, 17, 17, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.3200655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[1]R generation[0..12]: [2, 4, 5, 5, 11, 8, 11, 11, 11, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320079229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320093157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[2]R flush_numbers[0..12]: [3, 5, 8, 8, 17, 15, 17, 17, 17, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320106505Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[2]R generation[0..12]: [2, 4, 5, 5, 11, 8, 11, 11, 11, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320119803Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320133651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320146419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320159578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320172486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320185304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"":"downstairs","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320198262Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:50:48.32021305Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"0","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320240856Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"1","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320268392Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"client":"2","":"downstairs","session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320295829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320320585Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"b673c8f7-ea6a-4186-8b53-00fbf5045671 is now active with session: dd5118e7-03a1-43a4-bf56-14c52db18b60","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320347961Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320372968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229,"session_id":"dd5118e7-03a1-43a4-bf56-14c52db18b60"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:48.320399345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 406 len 36864 data: 4 4 4 4 3 3 2 2 2 02/30 Read block 326 len 20480 03/30 Write block 208 len 40960 data: 2 2 2 2 2 2 2 2 2 2 04/30 Write block 304 len 20480 data: 2 2 2 2 2 05/30 Write block 278 len 40960 data: 2 2 2 2 2 2 2 2 2 2 06/30 Write block 290 len 36864 data: 3 4 4 4 3 3 3 2 2 07/30 Write block 553 len 8192 data: 2 2 08/30 Read block 84 len 32768 09/30 Write block 301 len 20480 data: 2 2 2 3 3 10/30 Write block 291 len 12288 data: 5 5 5 11/30 Write block 110 len 8192 data: 3 4 12/30 Flush 13/30 Write block 99 len 16384 data: 2 2 2 2 14/30 Flush 15/30 Write block 488 len 28672 data: 2 2 2 2 2 2 2 16/30 Read block 99 len 32768 17/30 Read block 413 len 16384 18/30 Write block 515 len 24576 data: 4 3 3 2 3 3 19/30 Read block 516 len 40960 20/30 Write block 115 len 24576 data: 4 4 3 3 4 4 21/30 Read block 238 len 4096 22/30 Read block 366 len 8192 23/30 Write block 304 len 28672 data: 4 4 3 3 3 2 3 24/30 Read block 200 len 20480 25/30 Flush 26/30 Write block 389 len 36864 data: 2 2 2 2 2 2 2 2 2 27/30 Write block 85 len 20480 data: 3 3 3 3 3 28/30 Write block 42 len 32768 data: 2 2 2 3 3 3 3 3 29/30 Read block 42 len 16384 30/30 Read block 263 len 16384 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:48.347114194Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1229} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 5 5 5 8 8 8 T T T 3 060-079 5 5 5 8 8 8 F F F 4 080-099 12 12 12 19 19 19 T T T 5 100-119 12 12 12 20 20 20 F F F 6 120-139 12 12 12 20 20 20 F F F 7 140-159 11 11 11 17 17 17 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 12 12 12 18 18 18 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 10 10 10 16 16 16 F F F 17 340-359 11 11 11 17 17 17 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 5 5 5 8 8 8 T T T 20 400-419 12 12 12 18 18 18 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 12, Max flush: 20 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 12 12 12 20 20 20 F F F 6 120-139 12 12 12 20 20 20 F F F 7 140-159 11 11 11 17 17 17 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 12 12 12 18 18 18 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 10 10 10 16 16 16 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 10 10 10 16 16 16 F F F 17 340-359 11 11 11 17 17 17 F F F 18 360-379 5 5 5 8 8 8 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 12 12 12 18 18 18 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 13, Max flush: 21 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.631156732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.675905431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.676551092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.676594626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.676614363Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.676644699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237} {"msg":"Crucible da0dbfe3-8a87-4fb7-ba13-6f458ddb436c has session id: d6022876-dc47-4bf7-95cc-7a3b0a5b2020","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.677124233Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Upstairs opts: Upstairs UUID: da0dbfe3-8a87-4fb7-ba13-6f458ddb436c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.6772199Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Crucible stats registered with UUID: da0dbfe3-8a87-4fb7-ba13-6f458ddb436c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.677255165Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708398873Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237} {"msg":"da0dbfe3-8a87-4fb7-ba13-6f458ddb436c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708429318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708446396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708474252Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708497919Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708515926Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708532734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708549492Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.7085658Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708582697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708599215Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_connection connected from Ok(127.0.0.1:39902)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708625131Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_connection connected from Ok(127.0.0.1:37821)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708734756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_connection connected from Ok(127.0.0.1:46049)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.708937578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.709405923Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.709445738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.709477834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.70950455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.709526377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.709550854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710135493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710164259Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710180716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710742399Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710782293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710812019Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.71087946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710907756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710923034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710935582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.71094909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[0]R flush_numbers[0..12]: [3, 5, 21, 8, 21, 20, 20, 17, 17, 1, 18, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710962839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[0]R generation[0..12]: [2, 4, 13, 5, 13, 12, 12, 11, 11, 1, 12, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710976746Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.710990855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[1]R flush_numbers[0..12]: [3, 5, 21, 8, 21, 20, 20, 17, 17, 1, 18, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711009312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[1]R generation[0..12]: [2, 4, 13, 5, 13, 12, 12, 11, 11, 1, 12, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.71102421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711038348Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[2]R flush_numbers[0..12]: [3, 5, 21, 8, 21, 20, 20, 17, 17, 1, 18, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711051826Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[2]R generation[0..12]: [2, 4, 13, 5, 13, 12, 12, 11, 11, 1, 12, 8]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711065314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711079203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711091951Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711104929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711117697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711136575Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"":"downstairs","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711149813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711161651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"0","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711174739Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"1","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711187628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"client":"2","":"downstairs","session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711200596Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711214884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"da0dbfe3-8a87-4fb7-ba13-6f458ddb436c is now active with session: d6022876-dc47-4bf7-95cc-7a3b0a5b2020","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711228002Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.71123996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237,"session_id":"d6022876-dc47-4bf7-95cc-7a3b0a5b2020"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:52.711251939Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 588 len 32768 data: 2 2 2 3 3 2 2 2 03/30 Write block 102 len 36864 data: 3 2 3 3 3 4 4 3 4 04/30 Read block 526 len 32768 05/30 Read block 237 len 32768 06/30 Write block 317 len 24576 data: 2 2 2 2 2 2 07/30 Read block 427 len 28672 08/30 Flush 09/30 Write block 83 len 28672 data: 2 2 4 4 4 4 4 10/30 Read block 136 len 12288 11/30 Write block 224 len 20480 data: 2 2 2 2 2 12/30 Flush 13/30 Write block 246 len 36864 data: 2 2 2 2 2 3 2 2 2 14/30 Flush 15/30 Write block 301 len 32768 data: 3 3 3 5 5 4 4 4 16/30 Write block 388 len 24576 data: 2 3 3 3 3 3 17/30 Read block 327 len 32768 18/30 Write block 90 len 28672 data: 3 3 3 3 3 3 3 19/30 Read block 208 len 12288 20/30 Read block 406 len 32768 21/30 Flush 22/30 Write block 201 len 32768 data: 2 2 2 2 2 2 2 3 23/30 Write block 136 len 28672 data: 3 3 4 4 4 3 3 24/30 Read block 30 len 32768 25/30 Write block 225 len 20480 data: 3 3 3 3 2 26/30 Read block 156 len 12288 27/30 Write block 356 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Write block 246 len 20480 data: 3 3 3 3 3 29/30 Write block 139 len 12288 data: 5 5 4 30/30 Read block 288 len 8192 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:52.736325375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1237} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 12 12 12 20 20 20 T T T 7 140-159 11 11 11 17 17 17 T T T 8 160-179 11 11 11 17 17 17 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 12 12 12 18 18 18 T T T 11 220-239 14 14 14 24 24 24 T T T 12 240-259 14 14 14 25 25 25 T T T 13 260-279 12 12 12 18 18 18 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 14 14 14 23 23 23 F F F 17 340-359 11 11 11 17 17 17 T T T 18 360-379 5 5 5 8 8 8 T T T 19 380-399 14 14 14 26 26 26 F F F 20 400-419 12 12 12 18 18 18 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 14, Max flush: 26 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 4 4 4 5 5 5 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 5 5 5 8 8 8 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 15 15 15 27 27 27 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 14 14 14 23 23 23 F F F 17 340-359 15 15 15 27 27 27 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 12 12 12 18 18 18 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 15, Max flush: 27 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.063733253Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.108962695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.109802189Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.109851823Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.10987022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.109888547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245} {"msg":"Crucible 9383be74-bba2-46e0-b11e-4de1f95df6ce has session id: 12176526-470d-4eaf-b2a9-db32dbeb0858","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.110383579Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Upstairs opts: Upstairs UUID: 9383be74-bba2-46e0-b11e-4de1f95df6ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.110422394Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Crucible stats registered with UUID: 9383be74-bba2-46e0-b11e-4de1f95df6ce","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.110444431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141545245Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245} {"msg":"9383be74-bba2-46e0-b11e-4de1f95df6ce active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141581569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141599127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141627043Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141645111Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141662289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141678756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141702783Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.14172013Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141737738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141754626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_connection connected from Ok(127.0.0.1:43409)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.141896716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_connection connected from Ok(127.0.0.1:44117)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142205143Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_connection connected from Ok(127.0.0.1:56350)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142254987Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142698535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142727271Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142742949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142757137Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142778374Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.142804191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.143431204Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.14345981Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.14353178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144065446Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144091513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144106061Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144172472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144192629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144205967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144218385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144231683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[0]R flush_numbers[0..12]: [3, 5, 21, 8, 26, 23, 27, 27, 17, 1, 27, 27]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144251671Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[0]R generation[0..12]: [2, 4, 13, 5, 14, 14, 15, 15, 11, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144265899Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144280037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[1]R flush_numbers[0..12]: [3, 5, 21, 8, 26, 23, 27, 27, 17, 1, 27, 27]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144293775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[1]R generation[0..12]: [2, 4, 13, 5, 14, 14, 15, 15, 11, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144307443Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144321521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[2]R flush_numbers[0..12]: [3, 5, 21, 8, 26, 23, 27, 27, 17, 1, 27, 27]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144338799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[2]R generation[0..12]: [2, 4, 13, 5, 14, 14, 15, 15, 11, 1, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144352997Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144367365Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144380253Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144393341Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144406339Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144419528Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"":"downstairs","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144432566Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144444484Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"0","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:50:57.144457812Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"1","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.14447649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"client":"2","":"downstairs","session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144489838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144512695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"9383be74-bba2-46e0-b11e-4de1f95df6ce is now active with session: 12176526-470d-4eaf-b2a9-db32dbeb0858","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144526773Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144538991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245,"session_id":"12176526-470d-4eaf-b2a9-db32dbeb0858"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:50:57.144551189Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 314 len 24576 data: 3 2 2 3 3 3 02/30 Write block 523 len 16384 data: 3 3 3 3 03/30 Read block 575 len 16384 04/30 Read block 335 len 16384 05/30 Read block 338 len 16384 06/30 Read block 548 len 40960 07/30 Read block 4 len 16384 08/30 Read block 469 len 32768 09/30 Read block 340 len 20480 10/30 Read block 370 len 12288 11/30 Write block 221 len 16384 data: 3 3 2 3 12/30 Write block 224 len 24576 data: 4 4 4 4 4 3 13/30 Read block 141 len 4096 14/30 Read block 551 len 4096 15/30 Write block 27 len 28672 data: 2 2 2 2 2 2 3 16/30 Flush 17/30 Write block 2 len 20480 data: 2 2 2 2 2 18/30 Read block 318 len 20480 19/30 Read block 265 len 40960 20/30 Read block 103 len 28672 21/30 Flush 22/30 Write block 71 len 32768 data: 4 3 3 3 3 2 2 2 23/30 Read block 21 len 12288 24/30 Read block 467 len 16384 25/30 Read block 508 len 16384 26/30 Read block 458 len 32768 27/30 Write block 529 len 32768 data: 3 4 3 3 3 3 3 3 28/30 Write block 528 len 4096 data: 3 29/30 Write block 10 len 16384 data: 2 2 2 3 30/30 Write block 268 len 8192 data: 3 2 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 8 Sent Sent Sent false 28 Acked 1027 Write 1 Sent Sent Sent false 29 Acked 1028 Write 4 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:50:57.167146618Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1245} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 T T T 1 020-039 16 16 16 28 28 28 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 5 5 5 8 8 8 T T T 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 16 16 16 28 28 28 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 12 12 12 18 18 18 T T T 14 280-299 12 12 12 18 18 18 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 14 14 14 23 23 23 F F F 17 340-359 15 15 15 27 27 27 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 12 12 12 18 18 18 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 16 16 16 28 28 28 T T T 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 16, Max flush: 29 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 17 17 17 30 30 30 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 16 16 16 28 28 28 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 14 14 14 23 23 23 F F F 17 340-359 15 15 15 27 27 27 F F F 18 360-379 15 15 15 27 27 27 F F F 19 380-399 14 14 14 26 26 26 F F F 20 400-419 12 12 12 18 18 18 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 17, Max flush: 30 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.471850582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.515657221Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.516427174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.516472968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.516491146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.516523121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253} {"msg":"Crucible 0b45334e-c31c-487f-bf4c-a7470192cd44 has session id: 831de64f-9904-46b7-8596-4a465203ceea","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.516998396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Upstairs opts: Upstairs UUID: 0b45334e-c31c-487f-bf4c-a7470192cd44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.517034541Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Crucible stats registered with UUID: 0b45334e-c31c-487f-bf4c-a7470192cd44","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.517050648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548496565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253} {"msg":"0b45334e-c31c-487f-bf4c-a7470192cd44 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548536139Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548551157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548573374Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548587322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.54860068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548619518Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548649024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548680639Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548713285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548740111Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_connection connected from Ok(127.0.0.1:38255)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548907288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_connection connected from Ok(127.0.0.1:42554)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548948122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_connection connected from Ok(127.0.0.1:40352)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.548976209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.549945275Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.549972321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.549986879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.550000637Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.550014125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.550027283Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.550637789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.550666415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.550877846Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551201721Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551237606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551265232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551292169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551321434Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551347501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551448657Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551468224Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[0]R flush_numbers[0..12]: [30, 28, 21, 30, 26, 23, 27, 27, 17, 1, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551488811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[0]R generation[0..12]: [17, 16, 13, 17, 14, 14, 15, 15, 11, 1, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551503199Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551517247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[1]R flush_numbers[0..12]: [30, 28, 21, 30, 26, 23, 27, 27, 17, 1, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551530696Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[1]R generation[0..12]: [17, 16, 13, 17, 14, 14, 15, 15, 11, 1, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551544344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551558412Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[2]R flush_numbers[0..12]: [30, 28, 21, 30, 26, 23, 27, 27, 17, 1, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.55157187Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[2]R generation[0..12]: [17, 16, 13, 17, 14, 14, 15, 15, 11, 1, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551585588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551599446Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551612324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551628602Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.55164231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551655538Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"":"downstairs","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551668737Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA",Wait for a query_work_queue command to finish before sending IO "pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551689074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"0","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551702412Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"1","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.55171534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"client":"2","":"downstairs","session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551728608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551741016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"0b45334e-c31c-487f-bf4c-a7470192cd44 is now active with session: 831de64f-9904-46b7-8596-4a465203ceea","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551753725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551766053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253,"session_id":"831de64f-9904-46b7-8596-4a465203ceea"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:01.551778251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 457 len 8192 data: 3 3 02/30 Write block 348 len 20480 data: 5 4 4 4 4 03/30 Read block 210 len 12288 04/30 Read block 51 len 8192 05/30 Read block 422 len 24576 06/30 Read block 354 len 28672 07/30 Read block 390 len 12288 08/30 Read block 316 len 8192 09/30 Write block 373 len 32768 data: 2 2 2 3 3 3 3 3 10/30 Write block 319 len 32768 data: 4 3 3 3 2 2 3 3 11/30 Read block 493 len 20480 12/30 Write block 333 len 20480 data: 2 2 2 2 3 13/30 Read block 469 len 28672 14/30 Read block 558 len 12288 15/30 Write block 338 len 20480 data: 3 3 3 3 3 16/30 Flush 17/30 Write block 196 len 12288 data: 2 2 2 18/30 Write block 367 len 24576 data: 2 2 2 2 2 2 19/30 Read block 411 len 12288 20/30 Read block 15 len 16384 21/30 Flush 22/30 Write block 404 len 24576 data: 5 5 5 5 5 5 23/30 Write block 493 len 20480 data: 3 3 2 2 4 24/30 Read block 60 len 28672 25/30 Read block 540 len 40960 26/30 Read block 560 len 28672 27/30 Read block 216 len 16384 28/30 Read block 406 len 12288 29/30 Read block 405 len 12288 30/30 Read block 50 len 40960 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:01.576290605Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1253} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 17 17 17 30 30 30 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 18 18 18 32 32 32 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 16 16 16 28 28 28 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 18 18 18 31 31 31 F F F 16 320-339 18 18 18 31 31 31 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 18 18 18 31 31 31 F F F 20 400-419 12 12 12 18 18 18 T T T 21 420-439 11 11 11 17 17 17 F F F 22 440-459 18 18 18 31 31 31 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 12 12 12 20 20 20 T T T 25 500-519 12 12 12 20 20 20 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 18, Max flush: 32 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 17 17 17 30 30 30 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 18 18 18 32 32 32 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 16 16 16 28 28 28 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 18 18 18 31 31 31 F F F 16 320-339 18 18 18 31 31 31 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 18 18 18 31 31 31 F F F 20 400-419 19 19 19 33 33 33 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 18 18 18 31 31 31 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 19 19 19 33 33 33 F F F 25 500-519 12 12 12 20 20 20 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 19, Max flush: 33 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.85261879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:05.896819244Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.897442198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.930449279Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.931446061Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.931485005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.931500503Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.931515031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261} {"msg":"Crucible 37a3f076-47c2-4334-9e78-1106989dd08d has session id: 58132334-ac83-4c71-b9a5-b81ef1165881","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.932016312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Upstairs opts: Upstairs UUID: 37a3f076-47c2-4334-9e78-1106989dd08d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.932048488Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Crucible stats registered with UUID: 37a3f076-47c2-4334-9e78-1106989dd08d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.932063545Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964001114Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261} {"msg":"37a3f076-47c2-4334-9e78-1106989dd08d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964068904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964106599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964142244Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964166231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964191208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964215564Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964240651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964266907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964290814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964314661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_connection connected from Ok(127.0.0.1:39942)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964341927Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_connection connected from Ok(127.0.0.1:34813)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964368763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_connection connected from Ok(127.0.0.1:49388)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.964400869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.965211507Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.965244342Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.965271708Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.96532895Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.965364465Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.965381463Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.965865256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.965893732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.96590956Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966458784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966486661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966502948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966515666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966529075Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966547022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.9665599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966573389Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[0]R flush_numbers[0..12]: [30, 28, 21, 30, 26, 23, 27, 27, 17, 32, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966588076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[0]R generation[0..12]: [17, 16, 13, 17, 14, 14, 15, 15, 11, 18, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966601945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966616263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[1]R flush_numbers[0..12]: [30, 28, 21, 30, 26, 23, 27, 27, 17, 32, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966629931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[1]R generation[0..12]: [17, 16, 13, 17, 14, 14, 15, 15, 11, 18, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966643599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966657647Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[2]R flush_numbers[0..12]: [30, 28, 21, 30, 26, 23, 27, 27, 17, 32, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966671435Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[2]R generation[0..12]: [17, 16, 13, 17, 14, 14, 15, 15, 11, 18, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966685113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966699111Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.96671209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966725468Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966738476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966753844Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"":"downstairs","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966767452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Transition from WaitQuorum to Active","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966786379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"0","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966799647Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"1","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966812836Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"client":"2","":"downstairs","session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966826014Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966838282Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"37a3f076-47c2-4334-9e78-1106989dd08d is now active with session: 58132334-ac83-4c71-b9a5-b81ef1165881","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.96685112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966863229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261,"session_id":"58132334-ac83-4c71-b9a5-b81ef1165881"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:05.966875677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 548 len 8192 data: 5 3 02/30 Write block 65 len 20480 data: 3 3 3 3 4 03/30 Write block 540 len 28672 data: 5 6 4 4 4 4 5 04/30 Read block 74 len 36864 05/30 Read block 456 len 32768 06/30 Flush 07/30 Write block 210 len 20480 data: 3 3 3 3 3 08/30 Flush 09/30 Write block 271 len 8192 data: 3 3 10/30 Read block 288 len 40960 11/30 Flush 12/30 Write block 68 len 24576 data: 4 5 4 5 4 4 13/30 Write block 506 len 4096 data: 4 14/30 Write block 424 len 16384 data: 3 3 2 2 15/30 Read block 561 len 12288 16/30 Write block 390 len 28672 data: 4 4 4 4 3 3 3 17/30 Read block 459 len 12288 18/30 Read block 151 len 36864 19/30 Write block 216 len 24576 data: 3 3 3 3 3 4 20/30 Read block 428 len 36864 21/30 Read block 253 len 24576 22/30 Read block 191 len 36864 23/30 Write block 553 len 16384 data: 3 3 2 3 24/30 Flush 25/30 Write block 112 len 36864 data: 4 3 3 5 5 4 4 5 5 26/30 Write block 77 len 32768 data: 3 3 2 2 2 2 3 3 27/30 Read block 214 len 4096 28/30 Write block 181 len 40960 data: 2 2 2 2 2 2 2 2 2 2 29/30 Write block 459 len 28672 data: 3 2 3 3 3 3 3 30/30 Read block 527 len 36864 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:05.993798648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1261} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 20 20 20 37 37 37 T T T 4 080-099 14 14 14 26 26 26 T T T 5 100-119 14 14 14 23 23 23 T T T 6 120-139 15 15 15 27 27 27 T T T 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 18 18 18 32 32 32 T T T 10 200-219 20 20 20 37 37 37 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 20 20 20 36 36 36 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 18 18 18 31 31 31 F F F 16 320-339 18 18 18 31 31 31 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 20 20 20 37 37 37 F F F 20 400-419 19 19 19 33 33 33 F F F 21 420-439 20 20 20 37 37 37 F F F 22 440-459 18 18 18 31 31 31 T T T 23 460-479 7 7 7 14 14 14 T T T 24 480-499 19 19 19 33 33 33 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 20, Max flush: 37 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 21 21 21 38 38 38 F F F 4 080-099 21 21 21 38 38 38 F F F 5 100-119 21 21 21 38 38 38 F F F 6 120-139 21 21 21 38 38 38 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 21 21 21 38 38 38 F F F 10 200-219 20 20 20 37 37 37 F F F 11 220-239 20 20 20 37 37 37 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 20 20 20 36 36 36 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 18 18 18 31 31 31 F F F 16 320-339 18 18 18 31 31 31 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 20 20 20 37 37 37 F F F 20 400-419 19 19 19 33 33 33 F F F 21 420-439 20 20 20 37 37 37 F F F 22 440-459 21 21 21 38 38 38 F F F 23 460-479 21 21 21 38 38 38 F F F 24 480-499 19 19 19 33 33 33 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 F F F Max gen: 21, Max flush: 38 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.372735178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:10.416761947Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.417175379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.450046099Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.450799325Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.450842959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.450860057Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.450877934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269} {"msg":"Crucible 9e1f3a1f-3009-45c5-a122-1ae045f72377 has session id: 5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.451354438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Upstairs opts: Upstairs UUID: 9e1f3a1f-3009-45c5-a122-1ae045f72377, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.451402842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Crucible stats registered with UUID: 9e1f3a1f-3009-45c5-a122-1ae045f72377","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.451436957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484182864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269} {"msg":"9e1f3a1f-3009-45c5-a122-1ae045f72377 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484232117Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484248794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.48427851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484295478Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484309586Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484323944Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484338472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.48435282Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484368218Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484382676Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63865)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.4844974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48965)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484536945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49607)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.484562801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.485424652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.485463477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.485480834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.485496192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.48551007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.485524268Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486151582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486179858Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486195745Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486793953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486832378Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486857184Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486871452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.48688576Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486899608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486911797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486924815Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[0]R flush_numbers[0..12]: [30, 28, 21, 38, 38, 38, 38, 27, 17, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486944222Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[0]R generation[0..12]: [17, 16, 13, 21, 21, 21, 21, 15, 11, 21, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.48695977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486974118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[1]R flush_numbers[0..12]: [30, 28, 21, 38, 38, 38, 38, 27, 17, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.486988006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[1]R generation[0..12]: [17, 16, 13, 21, 21, 21, 21, 15, 11, 21, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487001494Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487015502Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[2]R flush_numbers[0..12]: [30, 28, 21, 38, 38, 38, 38, 27, 17, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487029301Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[2]R generation[0..12]: [17, 16, 13, 21, 21, 21, 21, 15, 11, 21, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487042648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487056537Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487069685Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487082733Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.48710215Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487115369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"":"downstairs","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487128667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487143075Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"0","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487156533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"1","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487169541Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"client":"2","":"downstairs","session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487182739Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487195087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"9e1f3a1f-3009-45c5-a122-1ae045f72377 is now active with session: 5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487208006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487220044Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269,"session_id":"5a4c5dcc-ce07-4d4c-87b0-9e3e726c252c"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:10.487232342Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 224 len 4096 data: 5 02/30 Write block 77 len 24576 data: 4 4 3 3 3 3 03/30 Read block 57 len 24576 04/30 Flush 05/30 Write block 375 len 12288 data: 3 4 4 06/30 Read block 238 len 8192 07/30 Read block 494 len 8192 08/30 Write block 196 len 20480 data: 3 3 3 2 2 09/30 Read block 357 len 12288 10/30 Write block 263 len 32768 data: 3 3 3 3 3 4 3 2 11/30 Write block 98 len 8192 data: 2 3 12/30 Read block 245 len 32768 13/30 Flush 14/30 Write block 582 len 4096 data: 2 15/30 Read block 560 len 32768 16/30 Write block 244 len 32768 data: 2 2 4 4 4 4 4 4 17/30 Read block 222 len 4096 18/30 Write block 221 len 28672 data: 5 4 3 6 5 5 5 19/30 Read block 535 len 40960 20/30 Read block 578 len 40960 21/30 Write block 42 len 20480 data: 3 3 3 4 4 22/30 Write block 459 len 20480 data: 4 3 4 4 4 23/30 Read block 306 len 28672 24/30 Read block 435 len 24576 25/30 Write block 309 len 20480 data: 3 4 3 3 3 26/30 Write block 249 len 28672 data: 5 5 5 3 3 3 2 27/30 Read block 203 len 36864 28/30 Write block 183 len 8192 data: 3 3 29/30 Read block 372 len 32768 30/30 Read block 552 len 4096 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 1 Done Done Done false 15 Acked 1014 Read 8 Done Done Done false 16 Acked 1015 Write 8 Done Done Done false 17 Acked 1016 Read 1 Done Done Done false 18 Acked 1017 Write 7 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:10.510781959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1269} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 13 13 13 21 21 21 T T T 3 060-079 22 22 22 39 39 39 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 21 21 21 38 38 38 F F F 6 120-139 21 21 21 38 38 38 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 22 22 22 40 40 40 T T T 10 200-219 22 22 22 40 40 40 F F F 11 220-239 22 22 22 39 39 39 T T T 12 240-259 15 15 15 27 27 27 T T T 13 260-279 22 22 22 40 40 40 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 18 18 18 31 31 31 T T T 16 320-339 18 18 18 31 31 31 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 20 20 20 37 37 37 F F F 20 400-419 19 19 19 33 33 33 F F F 21 420-439 20 20 20 37 37 37 F F F 22 440-459 21 21 21 38 38 38 T T T 23 460-479 21 21 21 38 38 38 T T T 24 480-499 19 19 19 33 33 33 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 14 14 14 23 23 23 T T T Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 22 22 22 39 39 39 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 21 21 21 38 38 38 F F F 6 120-139 21 21 21 38 38 38 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 22 22 22 40 40 40 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 23 23 23 41 41 41 F F F 16 320-339 18 18 18 31 31 31 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 20 20 20 37 37 37 F F F 20 400-419 19 19 19 33 33 33 F F F 21 420-439 20 20 20 37 37 37 F F F 22 440-459 23 23 23 41 41 41 F F F 23 460-479 23 23 23 41 41 41 F F F 24 480-499 19 19 19 33 33 33 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 23 23 23 41 41 41 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.893946425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.939067882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.939753977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.939786543Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.939800161Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.939814199Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277} {"msg":"Crucible b7d33a65-7650-4038-b2bf-f5fbb8899e92 has session id: 07065c64-5a7f-4ba2-b8e4-0e5663e87707","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.940261067Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Upstairs opts: Upstairs UUID: b7d33a65-7650-4038-b2bf-f5fbb8899e92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.940301812Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Crucible stats registered with UUID: b7d33a65-7650-4038-b2bf-f5fbb8899e92","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.940329668Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.97129814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277} {"msg":"b7d33a65-7650-4038-b2bf-f5fbb8899e92 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971326356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971346963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.9713686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971383298Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971397417Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971410915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971424223Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971437311Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971452229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971476535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_connection connected from Ok(127.0.0.1:59794)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971628814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_connection connected from Ok(127.0.0.1:44856)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971820248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_connection connected from Ok(127.0.0.1:54164)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.971858893Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.972432823Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.972460979Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.972476417Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.972494655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.972514392Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.972584573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.973171781Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.97325607Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.973304243Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.9738308Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.973859396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.973878924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.973952304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.97397941Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.973994178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974006966Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974020864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[0]R flush_numbers[0..12]: [30, 28, 41, 39, 40, 38, 38, 27, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974034432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[0]R generation[0..12]: [17, 16, 23, 22, 22, 21, 21, 15, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.97405217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974066848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[1]R flush_numbers[0..12]: [30, 28, 41, 39, 40, 38, 38, 27, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974080426Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[1]R generation[0..12]: [17, 16, 23, 22, 22, 21, 21, 15, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974093864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974107692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[2]R flush_numbers[0..12]: [30, 28, 41, 39, 40, 38, 38, 27, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.97412121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[2]R generation[0..12]: [17, 16, 23, 22, 22, 21, 21, 15, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974134548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974148516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974161365Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974174463Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974187321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974200319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"":"downstairs","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974213677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-09-27T03:51:14.974232355Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"0","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974245903Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"1","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974262081Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"client":"2","":"downstairs","session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974275939Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974288207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"b7d33a65-7650-4038-b2bf-f5fbb8899e92 is now active with session: 07065c64-5a7f-4ba2-b8e4-0e5663e87707","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974300815Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974312774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277,"session_id":"07065c64-5a7f-4ba2-b8e4-0e5663e87707"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:14.974324902Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 74 len 20480 data: 4 4 3 5 5 02/30 Write block 258 len 16384 data: 3 3 4 4 03/30 Write block 53 len 4096 data: 2 04/30 Read block 214 len 28672 05/30 Read block 109 len 16384 06/30 Write block 88 len 24576 data: 5 5 4 4 4 4 07/30 Write block 302 len 8192 data: 4 4 08/30 Write block 368 len 8192 data: 3 3 09/30 Read block 361 len 8192 10/30 Write block 582 len 28672 data: 3 2 2 2 2 2 3 11/30 Write block 436 len 8192 data: 2 2 12/30 Flush 13/30 Write block 279 len 40960 data: 3 3 3 3 3 3 3 3 3 2 14/30 Write block 270 len 40960 data: 3 4 4 3 3 3 3 3 3 4 15/30 Write block 134 len 32768 data: 5 3 4 4 5 6 6 5 16/30 Read block 308 len 16384 17/30 Read block 135 len 28672 18/30 Read block 410 len 20480 19/30 Flush 20/30 Write block 376 len 8192 data: 5 5 21/30 Write block 56 len 24576 data: 2 2 2 2 2 2 22/30 Read block 118 len 4096 23/30 Read block 127 len 24576 24/30 Read block 2 len 28672 25/30 Write block 318 len 40960 data: 4 5 4 4 4 3 3 4 4 3 26/30 Read block 420 len 16384 27/30 Read block 358 len 40960 28/30 Read block 429 len 20480 29/30 Write block 29 len 20480 data: 3 3 3 3 4 30/30 Read block 276 len 4096 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 5 Sent Done Sent false 30 Acked 1029 Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 9 11 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:14.997955568Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1277} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 16 16 16 28 28 28 T T T 2 040-059 24 24 24 42 42 42 T T T 3 060-079 24 24 24 42 42 42 T T T 4 080-099 24 24 24 42 42 42 F F F 5 100-119 21 21 21 38 38 38 F F F 6 120-139 24 24 24 43 43 43 F F F 7 140-159 24 24 24 43 43 43 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 43 43 43 F F F 14 280-299 24 24 24 43 43 43 F F F 15 300-319 24 24 24 42 42 42 T T T 16 320-339 18 18 18 31 31 31 T T T 17 340-359 18 18 18 31 31 31 F F F 18 360-379 24 24 24 42 42 42 T T T 19 380-399 20 20 20 37 37 37 F F F 20 400-419 19 19 19 33 33 33 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 23 23 23 41 41 41 F F F 23 460-479 23 23 23 41 41 41 F F F 24 480-499 19 19 19 33 33 33 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 24 24 24 42 42 42 F F F Max gen: 24, Max flush: 43 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 F F F 1 020-039 25 25 25 44 44 44 F F F 2 040-059 25 25 25 44 44 44 F F F 3 060-079 25 25 25 44 44 44 F F F 4 080-099 24 24 24 42 42 42 F F F 5 100-119 21 21 21 38 38 38 F F F 6 120-139 24 24 24 43 43 43 F F F 7 140-159 24 24 24 43 43 43 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 43 43 43 F F F 14 280-299 24 24 24 43 43 43 F F F 15 300-319 25 25 25 44 44 44 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 18 18 18 31 31 31 F F F 18 360-379 25 25 25 44 44 44 F F F 19 380-399 20 20 20 37 37 37 F F F 20 400-419 19 19 19 33 33 33 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 23 23 23 41 41 41 F F F 23 460-479 23 23 23 41 41 41 F F F 24 480-499 19 19 19 33 33 33 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 24 24 24 42 42 42 F F F Max gen: 25, Max flush: 44 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.319135896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.362846268Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.363423138Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.363459413Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.363473531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.363488009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285} {"msg":"Crucible d450e122-ca44-4032-880d-f0baf760b040 has session id: 9bb05dae-51d3-4bce-bd2c-cf582554497d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.363941107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Upstairs opts: Upstairs UUID: d450e122-ca44-4032-880d-f0baf760b040, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.363997109Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Crucible stats registered with UUID: d450e122-ca44-4032-880d-f0baf760b040","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.364022236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.395965713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285} {"msg":"d450e122-ca44-4032-880d-f0baf760b040 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396008787Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396026555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396054181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396072039Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396095525Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396114273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.39613117Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396157607Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396184063Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396212919Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38304)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396245735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47132)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396388355Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44196)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396443747Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.39692545Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.396964455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.397017238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.397048024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.39721819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.397257375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.397663419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.397693264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.397710082Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398294421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398322078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398336546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398349144Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398363652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.39837703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398389348Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398402706Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[0]R flush_numbers[0..12]: [30, 44, 44, 44, 42, 38, 43, 43, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398416415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[0]R generation[0..12]: [17, 25, 25, 25, 24, 21, 24, 24, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398430253Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398444521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[1]R flush_numbers[0..12]: [30, 44, 44, 44, 42, 38, 43, 43, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398458089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[1]R generation[0..12]: [17, 25, 25, 25, 24, 21, 24, 24, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398475207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398489645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[2]R flush_numbers[0..12]: [30, 44, 44, 44, 42, 38, 43, 43, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398503213Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[2]R generation[0..12]: [17, 25, 25, 25, 24, 21, 24, 24, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398516631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398530519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398543377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398556345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398569574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398600579Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"":"downstairs","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398627575Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398641544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"0","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398654962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"1","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.39866807Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"client":"2","":"downstairs","session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398681588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398693906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"d450e122-ca44-4032-880d-f0baf760b040 is now active with session: 9bb05dae-51d3-4bce-bd2c-cf582554497d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398709614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398721933Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285,"session_id":"9bb05dae-51d3-4bce-bd2c-cf582554497d"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:19.398734131Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 351 len 12288 data: 5 5 4 02/30 Write block 294 len 36864 data: 4 4 4 3 3 2 2 4 5 03/30 Flush 04/30 Write block 104 len 28672 data: 4 4 4 5 5 4 5 05/30 Write block 405 len 28672 data: 6 6 6 6 6 4 4 06/30 Flush 07/30 Write block 499 len 16384 data: 5 5 5 6 08/30 Read block 335 len 8192 09/30 Write block 452 len 36864 data: 3 3 4 4 3 4 4 5 4 10/30 Write block 67 len 8192 data: 4 5 11/30 Write block 259 len 28672 data: 4 5 5 5 4 4 4 12/30 Write block 76 len 4096 data: 4 13/30 Write block 379 len 24576 data: 4 4 3 2 2 2 14/30 Read block 191 len 20480 15/30 Write block 141 len 36864 data: 6 4 3 3 3 3 3 2 2 16/30 Read block 286 len 24576 17/30 Write block 77 len 16384 data: 6 6 4 4 18/30 Read block 238 len 20480 19/30 Write block 504 len 28672 data: 5 4 5 3 2 2 2 20/30 Read block 449 len 12288 21/30 Read block 199 len 8192 22/30 Write block 2 len 8192 data: 3 3 23/30 Write block 67 len 28672 data: 5 6 6 5 6 5 5 24/30 Read block 283 len 12288 25/30 Read block 209 len 36864 26/30 Write block 143 len 16384 data: 4 4 4 4 27/30 Write block 51 len 4096 data: 4 28/30 Write block 399 len 20480 data: 2 2 2 3 4 29/30 Write block 590 len 36864 data: 3 4 4 3 3 3 2 2 2 30/30 Write block 128 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 7 Acked 1006 Write 4 Done Done Done false 8 Acked 1007 Read 2 Done Done Done false 9 Acked 1008 Write 9 Done Done Done false 10 Acked 1009 Write 2 Done Done Done false 11 Acked 1010 Write 7 Done Done Done false 12 Acked 1011 Write 1 Done Done Done false 13 Acked 1012 Write 6 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Write 9 Done Done Done false 16 Acked 1015 Read 6 Done Done Done false 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Read 3 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:19.419749908Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1285} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 30 30 30 T T T 1 020-039 25 25 25 44 44 44 F F F 2 040-059 25 25 25 44 44 44 T T T 3 060-079 25 25 25 44 44 44 T T T 4 080-099 24 24 24 42 42 42 T T T 5 100-119 26 26 26 46 46 46 F F F 6 120-139 24 24 24 43 43 43 T T T 7 140-159 24 24 24 43 43 43 T T T 8 160-179 11 11 11 17 17 17 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 24 24 24 42 42 42 T T T 13 260-279 24 24 24 43 43 43 T T T 14 280-299 26 26 26 45 45 45 F F F 15 300-319 26 26 26 45 45 45 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 26 26 26 45 45 45 F F F 18 360-379 25 25 25 44 44 44 T T T 19 380-399 20 20 20 37 37 37 T T T 20 400-419 26 26 26 46 46 46 T T T 21 420-439 24 24 24 42 42 42 F F F 22 440-459 23 23 23 41 41 41 T T T 23 460-479 23 23 23 41 41 41 T T T 24 480-499 19 19 19 33 33 33 T T T 25 500-519 20 20 20 37 37 37 T T T 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 24 24 24 42 42 42 T T T Max gen: 26, Max flush: 46 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 47 47 47 F F F 1 020-039 25 25 25 44 44 44 F F F 2 040-059 27 27 27 47 47 47 F F F 3 060-079 27 27 27 47 47 47 F F F 4 080-099 27 27 27 47 47 47 F F F 5 100-119 26 26 26 46 46 46 F F F 6 120-139 27 27 27 47 47 47 F F F 7 140-159 27 27 27 47 47 47 F F F 8 160-179 11 11 11 17 17 17 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 23 23 23 41 41 41 F F F 12 240-259 27 27 27 47 47 47 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 26 26 26 45 45 45 F F F 15 300-319 26 26 26 45 45 45 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 26 26 26 45 45 45 F F F 18 360-379 27 27 27 47 47 47 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 27 27 27 47 47 47 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 27 27 27 47 47 47 F F F 23 460-479 27 27 27 47 47 47 F F F 24 480-499 27 27 27 47 47 47 F F F 25 500-519 27 27 27 47 47 47 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 11 11 11 17 17 17 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 27, Max flush: 47 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.87721701Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.922240651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.923129488Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.923167813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.923182141Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.923196589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293} {"msg":"Crucible 39d535c3-28ed-4966-9175-236a60ab659f has session id: 7d99a43b-8eee-43cc-9727-3f670aa42492","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.923753322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Upstairs opts: Upstairs UUID: 39d535c3-28ed-4966-9175-236a60ab659f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.923792756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Crucible stats registered with UUID: 39d535c3-28ed-4966-9175-236a60ab659f","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.923819093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956049171Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293} {"msg":"39d535c3-28ed-4966-9175-236a60ab659f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956083726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956098504Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956118341Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956136799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956151277Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956186972Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956216317Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956241754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956264931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956288058Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_connection connected from Ok(127.0.0.1:61727)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956388544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_connection connected from Ok(127.0.0.1:53722)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.956549631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_connection connected from Ok(127.0.0.1:62430)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.95663006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.957153398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.957222738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.957261043Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.957296128Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.957329904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.95736009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.957900835Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.957941009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958060033Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958386078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958414734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958430622Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958458068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958472536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958485924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958633354Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958652131Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[0]R flush_numbers[0..12]: [47, 44, 47, 47, 47, 46, 47, 47, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958676398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[0]R generation[0..12]: [27, 25, 27, 27, 27, 26, 27, 27, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958690566Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958704914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[1]R flush_numbers[0..12]: [47, 44, 47, 47, 47, 46, 47, 47, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958722071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[1]R generation[0..12]: [27, 25, 27, 27, 27, 26, 27, 27, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.95873612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958750307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[2]R flush_numbers[0..12]: [47, 44, 47, 47, 47, 46, 47, 47, 17, 41, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958763906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[2]R generation[0..12]: [27, 25, 27, 27, 27, 26, 27, 27, 11, 23, 22, 23]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958777314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958791002Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.95880465Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958817648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958830336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958843095Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"":"downstairs","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958856383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958868411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"0","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958881599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"1","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:51:23.958894628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"client":"2","":"downstairs","session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958913235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958927703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"39d535c3-28ed-4966-9175-236a60ab659f is now active with session: 7d99a43b-8eee-43cc-9727-3f670aa42492","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958940641Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958952659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293,"session_id":"7d99a43b-8eee-43cc-9727-3f670aa42492"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:23.958964788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 20 len 12288 data: 2 2 2 03/30 Read block 258 len 20480 04/30 Flush 05/30 Write block 432 len 8192 data: 2 2 06/30 Write block 475 len 4096 data: 3 07/30 Flush 08/30 Write block 234 len 4096 data: 2 09/30 Read block 330 len 24576 10/30 Write block 298 len 36864 data: 4 3 3 5 6 5 6 6 5 11/30 Read block 92 len 16384 12/30 Write block 563 len 28672 data: 2 3 3 4 4 4 4 13/30 Write block 408 len 40960 data: 7 7 5 5 3 3 3 2 2 3 14/30 Read block 31 len 36864 15/30 Write block 10 len 28672 data: 3 3 3 4 3 3 3 16/30 Read block 340 len 24576 17/30 Write block 14 len 12288 data: 4 4 4 18/30 Read block 472 len 32768 19/30 Write block 66 len 32768 data: 4 6 7 7 6 7 6 6 20/30 Flush 21/30 Write block 305 len 12288 data: 7 6 5 22/30 Read block 106 len 36864 23/30 Read block 111 len 36864 24/30 Write block 117 len 20480 data: 5 5 6 6 4 25/30 Write block 355 len 20480 data: 2 3 3 3 3 26/30 Write block 94 len 36864 data: 4 4 4 2 3 4 3 3 4 27/30 Read block 369 len 4096 28/30 Write block 176 len 8192 data: 2 2 29/30 Write block 181 len 32768 data: 3 3 4 4 3 3 3 3 30/30 Read block 196 len 36864 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Write 8 Done Sent Done false 30 Acked 1029 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 10 8 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:23.983584587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1293} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 51 51 51 F F F 1 020-039 28 28 28 49 49 49 F F F 2 040-059 27 27 27 47 47 47 F F F 3 060-079 28 28 28 51 51 51 F F F 4 080-099 27 27 27 47 47 47 T T T 5 100-119 26 26 26 46 46 46 T T T 6 120-139 27 27 27 47 47 47 T T T 7 140-159 27 27 27 47 47 47 F F F 8 160-179 11 11 11 17 17 17 T T T 9 180-199 23 23 23 41 41 41 T T T 10 200-219 22 22 22 40 40 40 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 27 27 27 47 47 47 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 28 28 28 51 51 51 T T T 16 320-339 25 25 25 44 44 44 F F F 17 340-359 26 26 26 45 45 45 T T T 18 360-379 27 27 27 47 47 47 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 28 28 28 51 51 51 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 27 27 27 47 47 47 F F F 23 460-479 28 28 28 50 50 50 F F F 24 480-499 27 27 27 47 47 47 F F F 25 500-519 27 27 27 47 47 47 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 28 28 28 51 51 51 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 28, Max flush: 51 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 51 51 51 F F F 1 020-039 28 28 28 49 49 49 F F F 2 040-059 27 27 27 47 47 47 F F F 3 060-079 28 28 28 51 51 51 F F F 4 080-099 29 29 29 52 52 52 F F F 5 100-119 29 29 29 52 52 52 F F F 6 120-139 29 29 29 52 52 52 F F F 7 140-159 27 27 27 47 47 47 F F F 8 160-179 29 29 29 52 52 52 F F F 9 180-199 29 29 29 52 52 52 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 27 27 27 47 47 47 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 29 29 29 52 52 52 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 29 29 29 52 52 52 F F F 18 360-379 27 27 27 47 47 47 F F F 19 380-399 27 27 27 47 47 47 F F F 20 400-419 28 28 28 51 51 51 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 27 27 27 47 47 47 F F F 23 460-479 28 28 28 50 50 50 F F F 24 480-499 27 27 27 47 47 47 F F F 25 500-519 27 27 27 47 47 47 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 28 28 28 51 51 51 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 29, Max flush: 52 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.328879184Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.374213822Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.375072433Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.375111258Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.375125866Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.375140414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301} {"msg":"Crucible 61c49f8a-be9f-4d8f-a4b8-953baa25106e has session id: 13dc115c-5621-4b4e-90c0-e95f7f94800b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.375664181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Upstairs opts: Upstairs UUID: 61c49f8a-be9f-4d8f-a4b8-953baa25106e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.375692257Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Crucible stats registered with UUID: 61c49f8a-be9f-4d8f-a4b8-953baa25106e","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.375706476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407263287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301} {"msg":"61c49f8a-be9f-4d8f-a4b8-953baa25106e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.40731456Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407335537Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407365852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.40738421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407401757Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407418825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407462509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407493735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407524651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407553816Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62015)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407584192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63508)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407613598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48501)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.407655932Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.408409598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.408443073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.40847044Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.408528992Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.408665973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.408707087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409212847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409252682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409287707Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409838651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409874276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409900152Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.40991698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409931128Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409944156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409956664Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409970673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[0]R flush_numbers[0..12]: [51, 49, 47, 51, 52, 52, 52, 47, 52, 52, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.409999549Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[0]R generation[0..12]: [28, 28, 27, 28, 29, 29, 29, 27, 29, 29, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410013916Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410028015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[1]R flush_numbers[0..12]: [51, 49, 47, 51, 52, 52, 52, 47, 52, 52, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410041623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[1]R generation[0..12]: [28, 28, 27, 28, 29, 29, 29, 27, 29, 29, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410055191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410069009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[2]R flush_numbers[0..12]: [51, 49, 47, 51, 52, 52, 52, 47, 52, 52, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410086746Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[2]R generation[0..12]: [28, 28, 27, 28, 29, 29, 29, 27, 29, 29, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410100565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410114413Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410127311Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:51:28.410140529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410222268Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410245915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"":"downstairs","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410260322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410274691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"0","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410299157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"1","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410324884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"client":"2","":"downstairs","session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.41035042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410365208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"61c49f8a-be9f-4d8f-a4b8-953baa25106e is now active with session: 13dc115c-5621-4b4e-90c0-e95f7f94800b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410384385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410406792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301,"session_id":"13dc115c-5621-4b4e-90c0-e95f7f94800b"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:28.410431469Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 455 len 12288 data: 5 4 5 02/30 Write block 347 len 36864 data: 5 6 5 5 6 6 5 2 3 03/30 Write block 191 len 4096 data: 2 04/30 Read block 53 len 24576 05/30 Flush 06/30 Write block 37 len 8192 data: 3 3 07/30 Flush 08/30 Write block 368 len 20480 data: 4 4 3 3 3 09/30 Read block 107 len 8192 10/30 Write block 500 len 16384 data: 6 6 7 6 11/30 Write block 381 len 8192 data: 4 3 12/30 Write block 148 len 32768 data: 3 3 2 3 3 3 3 3 13/30 Read block 16 len 4096 14/30 Read block 192 len 24576 15/30 Flush 16/30 Write block 479 len 12288 data: 2 2 2 17/30 Write block 135 len 36864 data: 4 5 5 6 7 7 7 5 5 18/30 Read block 372 len 40960 19/30 Write block 119 len 16384 data: 7 7 5 4 20/30 Write block 126 len 16384 data: 3 3 3 3 21/30 Write block 153 len 8192 data: 4 4 22/30 Write block 27 len 12288 data: 3 3 4 23/30 Read block 417 len 36864 24/30 Read block 136 len 8192 25/30 Write block 407 len 12288 data: 7 8 8 26/30 Write block 122 len 8192 data: 5 3 27/30 Read block 121 len 24576 28/30 Read block 441 len 40960 29/30 Read block 279 len 4096 30/30 Read block 446 len 28672 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 3 Done Done Done false 17 Acked 1016 Write 9 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:28.432546414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1301} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 51 51 51 F F F 1 020-039 30 30 30 54 54 54 T T T 2 040-059 27 27 27 47 47 47 F F F 3 060-079 28 28 28 51 51 51 F F F 4 080-099 29 29 29 52 52 52 F F F 5 100-119 29 29 29 52 52 52 T T T 6 120-139 29 29 29 52 52 52 T T T 7 140-159 30 30 30 55 55 55 T T T 8 160-179 29 29 29 52 52 52 F F F 9 180-199 30 30 30 53 53 53 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 27 27 27 47 47 47 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 29 29 29 52 52 52 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 30 30 30 53 53 53 F F F 18 360-379 30 30 30 55 55 55 F F F 19 380-399 30 30 30 55 55 55 F F F 20 400-419 28 28 28 51 51 51 T T T 21 420-439 28 28 28 50 50 50 F F F 22 440-459 30 30 30 53 53 53 F F F 23 460-479 28 28 28 50 50 50 T T T 24 480-499 27 27 27 47 47 47 T T T 25 500-519 30 30 30 55 55 55 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 28 28 28 51 51 51 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 30, Max flush: 55 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 51 51 51 F F F 1 020-039 31 31 31 56 56 56 F F F 2 040-059 27 27 27 47 47 47 F F F 3 060-079 28 28 28 51 51 51 F F F 4 080-099 29 29 29 52 52 52 F F F 5 100-119 31 31 31 56 56 56 F F F 6 120-139 31 31 31 56 56 56 F F F 7 140-159 31 31 31 56 56 56 F F F 8 160-179 29 29 29 52 52 52 F F F 9 180-199 30 30 30 53 53 53 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 27 27 27 47 47 47 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 29 29 29 52 52 52 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 30 30 30 53 53 53 F F F 18 360-379 30 30 30 55 55 55 F F F 19 380-399 30 30 30 55 55 55 F F F 20 400-419 31 31 31 56 56 56 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 30 30 30 53 53 53 F F F 23 460-479 31 31 31 56 56 56 F F F 24 480-499 31 31 31 56 56 56 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 28 28 28 51 51 51 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 31, Max flush: 56 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.778967476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.824916668Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.82584764Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.825886654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.825906702Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.82592189Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309} {"msg":"Crucible 61efad60-f6c6-4ad7-9497-098e2754ec87 has session id: c7197805-a6a9-4c03-826b-989bab51d0b5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.826317455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Upstairs opts: Upstairs UUID: 61efad60-f6c6-4ad7-9497-098e2754ec87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.82635533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Crucible stats registered with UUID: 61efad60-f6c6-4ad7-9497-098e2754ec87","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.826381356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859330155Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309} {"msg":"61efad60-f6c6-4ad7-9497-098e2754ec87 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859379378Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859397236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859428651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859446829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859464196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859480594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859497332Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.85951328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859530547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859556034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60885)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859679077Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:43956)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859732849Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33285)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.859767054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.860512591Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.860542178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.860558105Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.860573563Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.860755228Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.860789653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.86117233Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.861202176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.861506284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.86190075Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.861927546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.861947953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.861961182Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.861989707Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862004556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862016774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862030132Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[0]R flush_numbers[0..12]: [51, 56, 47, 51, 52, 56, 56, 56, 52, 53, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.86204386Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[0]R generation[0..12]: [28, 31, 27, 28, 29, 31, 31, 31, 29, 30, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862057688Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862071756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[1]R flush_numbers[0..12]: [51, 56, 47, 51, 52, 56, 56, 56, 52, 53, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862085334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[1]R generation[0..12]: [28, 31, 27, 28, 29, 31, 31, 31, 29, 30, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862098723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.86211249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[2]R flush_numbers[0..12]: [51, 56, 47, 51, 52, 56, 56, 56, 52, 53, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862125949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[2]R generation[0..12]: [28, 31, 27, 28, 29, 31, 31, 31, 29, 30, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862139367Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862157284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862170693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862183921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862197039Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862210007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"":"downstairs","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862223025Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862234904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"0","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862247882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","Wait for a query_work_queue command to finish before sending IO pid":1309,"client":"1","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862269699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"client":"2","":"downstairs","session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862282777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862294865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"61efad60-f6c6-4ad7-9497-098e2754ec87 is now active with session: c7197805-a6a9-4c03-826b-989bab51d0b5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862307514Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.862319472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309,"session_id":"c7197805-a6a9-4c03-826b-989bab51d0b5"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:32.86233164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 351 len 28672 data: 7 7 6 3 4 4 4 02/30 Write block 457 len 8192 data: 6 5 03/30 Write block 565 len 40960 data: 4 5 5 5 5 4 4 4 4 3 04/30 Write block 119 len 36864 data: 8 8 6 6 4 3 3 4 4 05/30 Read block 207 len 16384 06/30 Read block 199 len 36864 07/30 Read block 422 len 40960 08/30 Read block 123 len 8192 09/30 Write block 145 len 20480 data: 5 5 4 4 4 10/30 Read block 463 len 8192 11/30 Write block 43 len 36864 data: 4 4 5 5 4 4 4 3 5 12/30 Flush 13/30 Write block 72 len 36864 data: 7 7 5 5 5 7 7 5 5 14/30 Read block 4 len 36864 15/30 Read block 488 len 16384 16/30 Write block 305 len 12288 data: 8 7 6 17/30 Flush 18/30 Write block 443 len 24576 data: 2 2 2 2 2 2 19/30 Read block 283 len 16384 20/30 Read block 28 len 20480 21/30 Read block 127 len 36864 22/30 Read block 37 len 16384 23/30 Read block 410 len 24576 24/30 Read block 411 len 20480 25/30 Flush 26/30 Write block 69 len 36864 data: 8 7 8 8 8 6 6 6 8 27/30 Write block 174 len 40960 data: 2 2 3 3 2 2 2 4 4 5 28/30 Read block 18 len 4096 29/30 Write block 372 len 36864 data: 4 3 3 4 6 6 4 5 5 30/30 Read block 338 len 24576 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 9 Done Done Sent false 30 Acked 1029 Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 5 5 3 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:32.890188942Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1309} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 51 51 51 F F F 1 020-039 31 31 31 56 56 56 F F F 2 040-059 32 32 32 57 57 57 F F F 3 060-079 32 32 32 58 58 58 T T T 4 080-099 32 32 32 58 58 58 F F F 5 100-119 32 32 32 57 57 57 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 32 32 32 57 57 57 F F F 8 160-179 29 29 29 52 52 52 T T T 9 180-199 30 30 30 53 53 53 T T T 10 200-219 22 22 22 40 40 40 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 27 27 27 47 47 47 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 30 30 30 55 55 55 T T T 19 380-399 30 30 30 55 55 55 T T T 20 400-419 31 31 31 56 56 56 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 31 31 31 56 56 56 F F F 24 480-499 31 31 31 56 56 56 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 32 32 32 57 57 57 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 32, Max flush: 59 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 51 51 51 F F F 1 020-039 31 31 31 56 56 56 F F F 2 040-059 32 32 32 57 57 57 F F F 3 060-079 33 33 33 60 60 60 F F F 4 080-099 32 32 32 58 58 58 F F F 5 100-119 32 32 32 57 57 57 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 32 32 32 57 57 57 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 27 27 27 47 47 47 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 28 28 28 51 51 51 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 33 33 33 60 60 60 F F F 19 380-399 33 33 33 60 60 60 F F F 20 400-419 31 31 31 56 56 56 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 31 31 31 56 56 56 F F F 24 480-499 31 31 31 56 56 56 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 32 32 32 57 57 57 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 33, Max flush: 60 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.203883504Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.24879792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.24959557Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.249631975Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.249647613Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.249662271Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317} {"msg":"Crucible 9f505f23-7665-41ea-81e4-205e9e78c9fc has session id: 1f291782-702b-4a71-be28-22592e9d0c46","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.250146414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Upstairs opts: Upstairs UUID: 9f505f23-7665-41ea-81e4-205e9e78c9fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.250186578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Crucible stats registered with UUID: 9f505f23-7665-41ea-81e4-205e9e78c9fc","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.250218364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282423495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317} {"msg":"9f505f23-7665-41ea-81e4-205e9e78c9fc active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282451331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282466719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282486746Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282503194Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282529491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282556627Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282585053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.28260882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282624457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282639805Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_connection connected from Ok(127.0.0.1:45350)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.282777626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_connection connected from Ok(127.0.0.1:55977)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.28282383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_connection connected from Ok(127.0.0.1:49202)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.283054208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.283514675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.283554759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.283755771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.283798125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.283851538Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.283881474Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.284252113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.28441661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.284660056Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.284811925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.28485108Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.284876436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.284985691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285012237Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285027206Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285183854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285204561Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[0]R flush_numbers[0..12]: [51, 56, 57, 60, 58, 57, 57, 57, 60, 60, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285228928Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[0]R generation[0..12]: [28, 31, 32, 33, 32, 32, 32, 32, 33, 33, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285243316Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285257624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[1]R flush_numbers[0..12]: [51, 56, 57, 60, 58, 57, 57, 57, 60, 60, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285271252Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[1]R generation[0..12]: [28, 31, 32, 33, 32, 32, 32, 32, 33, 33, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.28528484Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285298958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[2]R flush_numbers[0..12]: [51, 56, 57, 60, 58, 57, 57, 57, 60, 60, 40, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285312526Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[2]R generation[0..12]: [28, 31, 32, 33, 32, 32, 32, 32, 33, 33, 22, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285325914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285339692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.28535249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285365509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285378577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285391605Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"":"downstairs","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285408823Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285421431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"0","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285434589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"1","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-09-27T03:51:37.285457936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"client":"2","":"downstairs","session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285471324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285483603Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"9f505f23-7665-41ea-81e4-205e9e78c9fc is now active with session: 1f291782-702b-4a71-be28-22592e9d0c46","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285496451Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285508499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317,"session_id":"1f291782-702b-4a71-be28-22592e9d0c46"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:37.285520837Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 477 len 36864 data: 2 2 3 3 3 2 2 2 2 02/30 Write block 156 len 4096 data: 3 03/30 Read block 132 len 28672 04/30 Write block 410 len 4096 data: 6 05/30 Read block 189 len 32768 06/30 Read block 512 len 36864 07/30 Read block 499 len 28672 08/30 Flush 09/30 Write block 252 len 24576 data: 4 4 4 3 2 2 10/30 Read block 207 len 40960 11/30 Flush 12/30 Write block 98 len 24576 data: 4 5 4 4 5 3 13/30 Read block 83 len 36864 14/30 Read block 168 len 28672 15/30 Read block 32 len 40960 16/30 Write block 513 len 24576 data: 3 4 5 4 4 3 17/30 Read block 427 len 24576 18/30 Write block 490 len 36864 data: 3 3 3 4 4 3 3 5 4 19/30 Read block 84 len 12288 20/30 Write block 240 len 40960 data: 2 2 2 2 3 3 5 5 5 6 21/30 Read block 151 len 16384 22/30 Flush 23/30 Write block 34 len 40960 data: 3 3 3 4 4 3 3 3 4 5 24/30 Write block 247 len 36864 data: 6 6 7 6 6 5 5 5 4 25/30 Write block 287 len 8192 data: 4 3 26/30 Write block 8 len 40960 data: 2 2 4 4 4 5 5 5 5 2 27/30 Write block 388 len 12288 data: 3 4 5 28/30 Write block 213 len 20480 data: 4 4 3 4 4 29/30 Read block 490 len 4096 30/30 Read block 88 len 40960 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:37.310870206Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1317} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 51 51 51 T T T 1 020-039 31 31 31 56 56 56 T T T 2 040-059 32 32 32 57 57 57 T T T 3 060-079 33 33 33 60 60 60 F F F 4 080-099 34 34 34 63 63 63 F F F 5 100-119 34 34 34 63 63 63 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 22 22 22 40 40 40 T T T 11 220-239 28 28 28 51 51 51 F F F 12 240-259 34 34 34 63 63 63 T T T 13 260-279 27 27 27 47 47 47 F F F 14 280-299 28 28 28 51 51 51 T T T 15 300-319 32 32 32 58 58 58 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 33 33 33 60 60 60 F F F 19 380-399 33 33 33 60 60 60 T T T 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 34 34 34 63 63 63 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 32 32 32 57 57 57 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 34, Max flush: 63 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 64 64 64 F F F 1 020-039 35 35 35 64 64 64 F F F 2 040-059 35 35 35 64 64 64 F F F 3 060-079 33 33 33 60 60 60 F F F 4 080-099 34 34 34 63 63 63 F F F 5 100-119 34 34 34 63 63 63 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 35 35 35 64 64 64 F F F 11 220-239 28 28 28 51 51 51 F F F 12 240-259 35 35 35 64 64 64 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 35 35 35 64 64 64 F F F 15 300-319 32 32 32 58 58 58 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 33 33 33 60 60 60 F F F 19 380-399 35 35 35 64 64 64 F F F 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 34 34 34 61 61 61 F F F 24 480-499 34 34 34 63 63 63 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 32 32 32 57 57 57 F F F 29 580-599 27 27 27 47 47 47 F F F Max gen: 35, Max flush: 64 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.65387288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.699388613Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.700230146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.700283169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.700307826Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.700324923Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325} {"msg":"Crucible 0639c65d-9a5c-4ec4-b850-32830e8354b2 has session id: 71e5e318-d05e-4604-ac69-ecfe613ba491","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.700734687Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Upstairs opts: Upstairs UUID: 0639c65d-9a5c-4ec4-b850-32830e8354b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.700774691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Crucible stats registered with UUID: 0639c65d-9a5c-4ec4-b850-32830e8354b2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.700798028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.732963995Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325} {"msg":"0639c65d-9a5c-4ec4-b850-32830e8354b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733028496Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.73306866Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733102825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733122273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.73313959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733165887Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733196463Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733223809Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733253095Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733306477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_connection connected from Ok(127.0.0.1:44992)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733340792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_connection connected from Ok(127.0.0.1:44648)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733370419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_connection connected from Ok(127.0.0.1:37061)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.733400284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734162309Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734195125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734211432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.73422558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734361951Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734405556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734794862Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734823378Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.734928843Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735422745Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735447112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735461659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735474408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735487916Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735501084Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735513912Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.73552729Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[0]R flush_numbers[0..12]: [64, 64, 64, 60, 63, 63, 57, 61, 60, 60, 64, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735540969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[0]R generation[0..12]: [35, 35, 35, 33, 34, 34, 32, 34, 33, 33, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735554557Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735573674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[1]R flush_numbers[0..12]: [64, 64, 64, 60, 63, 63, 57, 61, 60, 60, 64, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735588402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[1]R generation[0..12]: [35, 35, 35, 33, 34, 34, 32, 34, 33, 33, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.73560196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735615748Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[2]R flush_numbers[0..12]: [64, 64, 64, 60, 63, 63, 57, 61, 60, 60, 64, 51]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735629216Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[2]R generation[0..12]: [35, 35, 35, 33, 34, 34, 32, 34, 33, 33, 35, 28]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735642625Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735656383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735669401Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735682569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735695337Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735708456Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"":"downstairs","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735722014Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735734042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"0","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.73574724Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"1","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-09-27T03:51:41.735760398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"client":"2","":"downstairs","session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735785235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735797513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"0639c65d-9a5c-4ec4-b850-32830e8354b2 is now active with session: 71e5e318-d05e-4604-ac69-ecfe613ba491","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735810242Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.73582245Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325,"session_id":"71e5e318-d05e-4604-ac69-ecfe613ba491"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:41.735834688Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 40960 data: 4 4 4 3 3 3 5 6 7 6 02/30 Write block 46 len 32768 data: 6 5 5 5 4 6 3 3 03/30 Read block 285 len 8192 04/30 Read block 37 len 8192 05/30 Read block 574 len 12288 06/30 Read block 212 len 28672 07/30 Write block 369 len 20480 data: 5 4 4 5 4 08/30 Write block 579 len 36864 data: 2 2 2 4 3 3 3 3 3 09/30 Flush 10/30 Write block 229 len 8192 data: 4 2 11/30 Read block 249 len 28672 12/30 Read block 243 len 32768 13/30 Write block 544 len 16384 data: 5 5 6 5 14/30 Flush 15/30 Write block 282 len 28672 data: 4 4 4 4 4 5 4 16/30 Read block 540 len 24576 17/30 Write block 542 len 24576 data: 5 5 6 6 7 6 18/30 Write block 2 len 36864 data: 4 4 3 3 3 2 3 3 5 19/30 Write block 147 len 16384 data: 5 5 5 3 20/30 Read block 186 len 20480 21/30 Write block 468 len 8192 data: 2 2 22/30 Read block 5 len 8192 23/30 Read block 202 len 16384 24/30 Read block 489 len 8192 25/30 Read block 430 len 36864 26/30 Read block 49 len 40960 27/30 Write block 317 len 4096 data: 4 28/30 Read block 209 len 16384 29/30 Read block 325 len 28672 30/30 Write block 346 len 40960 data: 6 7 8 7 6 8 8 7 4 5 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 15 Acked 1014 Write 7 Done Done Done false 16 Acked 1015 Read 6 Done Done Done false 17 Acked 1016 Write 6 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:41.759983672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1325} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 64 64 64 T T T 1 020-039 35 35 35 64 64 64 F F F 2 040-059 36 36 36 65 65 65 F F F 3 060-079 33 33 33 60 60 60 F F F 4 080-099 34 34 34 63 63 63 F F F 5 100-119 34 34 34 63 63 63 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 34 34 34 61 61 61 T T T 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 35 35 35 64 64 64 F F F 11 220-239 36 36 36 66 66 66 F F F 12 240-259 35 35 35 64 64 64 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 35 35 35 64 64 64 T T T 15 300-319 32 32 32 58 58 58 T T T 16 320-339 25 25 25 44 44 44 F F F 17 340-359 36 36 36 65 65 65 T T T 18 360-379 36 36 36 65 65 65 F F F 19 380-399 35 35 35 64 64 64 F F F 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 34 34 34 61 61 61 T T T 24 480-499 34 34 34 63 63 63 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 36 36 36 66 66 66 T T T 28 560-579 36 36 36 65 65 65 F F F 29 580-599 36 36 36 65 65 65 F F F Max gen: 36, Max flush: 66 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 67 67 67 F F F 1 020-039 35 35 35 64 64 64 F F F 2 040-059 36 36 36 65 65 65 F F F 3 060-079 33 33 33 60 60 60 F F F 4 080-099 34 34 34 63 63 63 F F F 5 100-119 34 34 34 63 63 63 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 37 37 37 67 67 67 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 35 35 35 64 64 64 F F F 11 220-239 36 36 36 66 66 66 F F F 12 240-259 35 35 35 64 64 64 F F F 13 260-279 27 27 27 47 47 47 F F F 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 37 37 37 67 67 67 F F F 18 360-379 36 36 36 65 65 65 F F F 19 380-399 35 35 35 64 64 64 F F F 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 37 37 37 67 67 67 F F F 24 480-499 34 34 34 63 63 63 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 36 36 36 65 65 65 F F F 29 580-599 36 36 36 65 65 65 F F F Max gen: 37, Max flush: 67 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.104293845Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.149427321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.150066222Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.150102447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.150116536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.150130834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333} {"msg":"Crucible 096fdbd5-5892-4b31-8b52-e359552ff41c has session id: fb263621-4305-4fcf-9a30-1f43bd8d085b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.150527219Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Upstairs opts: Upstairs UUID: 096fdbd5-5892-4b31-8b52-e359552ff41c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.150561034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Crucible stats registered with UUID: 096fdbd5-5892-4b31-8b52-e359552ff41c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.150576752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182178327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333} {"msg":"096fdbd5-5892-4b31-8b52-e359552ff41c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182221031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182252337Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.18229909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182342014Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182375719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182410005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.18244537Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182697065Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182752757Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182784353Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55326)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182811379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59563)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182836116Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62267)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.182926683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.183436833Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.183464789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.183479997Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.183493745Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.183507543Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.183520711Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184219775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.18424915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184265318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184869375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184893992Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184908039Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184920688Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184934126Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184947414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184959432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184972651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[0]R flush_numbers[0..12]: [67, 64, 65, 60, 63, 63, 57, 67, 60, 60, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.184999567Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[0]R generation[0..12]: [37, 35, 36, 33, 34, 34, 32, 37, 33, 33, 35, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185013695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185031193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[1]R flush_numbers[0..12]: [67, 64, 65, 60, 63, 63, 57, 67, 60, 60, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.18505349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[1]R generation[0..12]: [37, 35, 36, 33, 34, 34, 32, 37, 33, 33, 35, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185071297Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185088845Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[2]R flush_numbers[0..12]: [67, 64, 65, 60, 63, 63, 57, 67, 60, 60, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185105532Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[2]R generation[0..12]: [37, 35, 36, 33, 34, 34, 32, 37, 33, 33, 35, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.18512246Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185139947Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185155905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185172183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.18519457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185210708Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"":"downstairs","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185227235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185242093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"0","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185258881Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"1","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185275369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"client":"2","":"downstairs","session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185291966Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185310054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"096fdbd5-5892-4b31-8b52-e359552ff41c is now active with session: fb263621-4305-4fcf-9a30-1f43bd8d085b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185326202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.18534097Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333,"session_id":"fb263621-4305-4fcf-9a30-1f43bd8d085b"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:46.185356398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 344 len 24576 data: 4 4 7 8 9 8 03/30 Write block 36 len 32768 data: 4 5 5 4 4 4 5 6 04/30 Write block 76 len 8192 data: 7 9 05/30 Write block 261 len 16384 data: 6 6 5 5 06/30 Write block 489 len 12288 data: 3 4 4 07/30 Read block 148 len 36864 08/30 Read block 534 len 24576 09/30 Write block 30 len 20480 data: 4 4 4 5 4 10/30 Write block 252 len 12288 data: 6 6 6 11/30 Read block 44 len 36864 12/30 Flush 13/30 Write block 522 len 16384 data: 3 4 4 4 14/30 Flush 15/30 Write block 261 len 20480 data: 7 7 6 6 5 16/30 Write block 206 len 8192 data: 3 3 17/30 Write block 40 len 40960 data: 5 5 6 7 5 6 7 6 6 6 18/30 Read block 190 len 16384 19/30 Read block 433 len 16384 20/30 Write block 90 len 8192 data: 5 5 21/30 Read block 28 len 16384 22/30 Read block 48 len 8192 23/30 Read block 316 len 28672 24/30 Read block 74 len 40960 25/30 Read block 402 len 20480 26/30 Read block 400 len 4096 27/30 Write block 61 len 20480 data: 3 2 3 3 4 28/30 Read block 61 len 4096 29/30 Read block 372 len 8192 30/30 Write block 369 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 15 Acked 1014 Write 5 Done Done Done false 16 Acked 1015 Write 2 Done Done Done false 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Read 4 Done Done Done false 19 Acked 1018 Read 4 Done Done Done false 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:46.207142158Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1333} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 67 67 67 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 38 38 38 69 69 69 T T T 3 060-079 38 38 38 69 69 69 T T T 4 080-099 34 34 34 63 63 63 T T T 5 100-119 34 34 34 63 63 63 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 37 37 37 67 67 67 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 35 35 35 64 64 64 T T T 11 220-239 36 36 36 66 66 66 F F F 12 240-259 38 38 38 69 69 69 F F F 13 260-279 38 38 38 69 69 69 T T T 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 38 38 38 69 69 69 F F F 18 360-379 36 36 36 65 65 65 T T T 19 380-399 35 35 35 64 64 64 F F F 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 37 37 37 67 67 67 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 38 38 38 70 70 70 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 36 36 36 65 65 65 F F F 29 580-599 36 36 36 65 65 65 F F F Max gen: 38, Max flush: 70 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 67 67 67 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 39 39 39 71 71 71 F F F 3 060-079 39 39 39 71 71 71 F F F 4 080-099 39 39 39 71 71 71 F F F 5 100-119 34 34 34 63 63 63 F F F 6 120-139 32 32 32 57 57 57 F F F 7 140-159 37 37 37 67 67 67 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 39 39 39 71 71 71 F F F 11 220-239 36 36 36 66 66 66 F F F 12 240-259 38 38 38 69 69 69 F F F 13 260-279 39 39 39 71 71 71 F F F 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 38 38 38 69 69 69 F F F 18 360-379 39 39 39 71 71 71 F F F 19 380-399 35 35 35 64 64 64 F F F 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 32 32 32 59 59 59 F F F 23 460-479 37 37 37 67 67 67 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 38 38 38 70 70 70 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 36 36 36 65 65 65 F F F 29 580-599 36 36 36 65 65 65 F F F Max gen: 39, Max flush: 71 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.536765231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.581751077Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.582417015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.58245315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.582467968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.582483726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341} {"msg":"Crucible ab5f14a3-c984-4232-ae77-b7c43d908fa3 has session id: 8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.582933664Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Upstairs opts: Upstairs UUID: ab5f14a3-c984-4232-ae77-b7c43d908fa3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.582974888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Crucible stats registered with UUID: ab5f14a3-c984-4232-ae77-b7c43d908fa3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.583000565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614072642Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341} {"msg":"ab5f14a3-c984-4232-ae77-b7c43d908fa3 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614108188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614130684Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614175588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614211853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614247508Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614290142Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614329357Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614365722Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614398108Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614428323Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_connection connected from Ok(127.0.0.1:38913)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614456369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_connection connected from Ok(127.0.0.1:44023)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.614560325Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_connection connected from Ok(127.0.0.1:42464)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.61459445Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.615132536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.615160152Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.61517732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.615191318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.615206426Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.615220044Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.615985618Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616106541Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616176882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616608552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616647147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616676513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616740164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.61676821Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616783178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616795956Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616810554Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[0]R flush_numbers[0..12]: [67, 69, 71, 71, 71, 63, 57, 67, 60, 60, 71, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616824392Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[0]R generation[0..12]: [37, 38, 39, 39, 39, 34, 32, 37, 33, 33, 39, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.61683839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616852758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[1]R flush_numbers[0..12]: [67, 69, 71, 71, 71, 63, 57, 67, 60, 60, 71, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616866667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[1]R generation[0..12]: [37, 38, 39, 39, 39, 34, 32, 37, 33, 33, 39, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616880254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616898662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[2]R flush_numbers[0..12]: [67, 69, 71, 71, 71, 63, 57, 67, 60, 60, 71, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.61691283Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[2]R generation[0..12]: [37, 38, 39, 39, 39, 34, 32, 37, 33, 33, 39, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616926658Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616940896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616954184Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616967493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.616980341Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617015076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"":"downstairs","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617030524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617042962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"0","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.61705724Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"1","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617070498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"client":"2","":"downstairs","session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617084026Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617096665Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"ab5f14a3-c984-4232-ae77-b7c43d908fa3 is now active with session: 8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617109523Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617123981Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341,"session_id":"8cb58ed0-7c19-4fd0-bd21-7ee0e0d7a5db"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:50.617136529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 563 len 12288 data: 3 4 5 02/30 Read block 458 len 4096 03/30 Write block 69 len 28672 data: 9 8 9 9 9 7 7 04/30 Read block 299 len 20480 05/30 Read block 58 len 20480 06/30 Write block 474 len 4096 data: 3 07/30 Write block 470 len 8192 data: 3 3 08/30 Read block 530 len 28672 09/30 Flush 10/30 Write block 453 len 20480 data: 4 5 6 5 7 11/30 Write block 482 len 24576 data: 3 3 3 3 2 2 12/30 Read block 430 len 16384 13/30 Read block 35 len 32768 14/30 Flush 15/30 Write block 449 len 12288 data: 2 2 2 16/30 Write block 377 len 16384 data: 7 5 6 6 17/30 Write block 124 len 40960 data: 4 4 5 5 4 4 5 5 5 5 18/30 Write block 577 len 32768 data: 3 3 3 3 3 5 4 4 19/30 Read block 567 len 8192 20/30 Write block 7 len 16384 data: 3 4 4 6 21/30 Read block 511 len 28672 22/30 Read block 500 len 12288 23/30 Read block 480 len 12288 24/30 Read block 172 len 4096 25/30 Read block 224 len 36864 26/30 Read block 269 len 40960 27/30 Write block 136 len 16384 data: 6 6 7 8 28/30 Read block 16 len 12288 29/30 Write block 388 len 4096 data: 4 30/30 Write block 528 len 36864 data: 4 4 5 4 4 4 4 4 4 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 15 Acked 1014 Write 3 Done Done Done false 16 Acked 1015 Write 4 Done Done Done false 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:50.640016799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1341} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 67 67 67 T T T 1 020-039 38 38 38 69 69 69 F F F 2 040-059 39 39 39 71 71 71 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 39 39 39 71 71 71 F F F 5 100-119 34 34 34 63 63 63 F F F 6 120-139 32 32 32 57 57 57 T T T 7 140-159 37 37 37 67 67 67 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 39 39 39 71 71 71 F F F 11 220-239 36 36 36 66 66 66 F F F 12 240-259 38 38 38 69 69 69 F F F 13 260-279 39 39 39 71 71 71 F F F 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 38 38 38 69 69 69 F F F 18 360-379 39 39 39 71 71 71 T T T 19 380-399 35 35 35 64 64 64 T T T 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 40 40 40 73 73 73 T T T 23 460-479 40 40 40 72 72 72 F F F 24 480-499 40 40 40 73 73 73 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 38 38 38 70 70 70 T T T 27 540-559 37 37 37 67 67 67 F F F 28 560-579 40 40 40 72 72 72 T T T 29 580-599 36 36 36 65 65 65 T T T Max gen: 40, Max flush: 73 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 39 39 39 71 71 71 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 39 39 39 71 71 71 F F F 5 100-119 34 34 34 63 63 63 F F F 6 120-139 41 41 41 74 74 74 F F F 7 140-159 37 37 37 67 67 67 F F F 8 160-179 33 33 33 60 60 60 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 39 39 39 71 71 71 F F F 11 220-239 36 36 36 66 66 66 F F F 12 240-259 38 38 38 69 69 69 F F F 13 260-279 39 39 39 71 71 71 F F F 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 38 38 38 69 69 69 F F F 18 360-379 41 41 41 74 74 74 F F F 19 380-399 41 41 41 74 74 74 F F F 20 400-419 34 34 34 61 61 61 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 41 41 41 74 74 74 F F F 23 460-479 40 40 40 72 72 72 F F F 24 480-499 40 40 40 73 73 73 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 41 41 41 74 74 74 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 41 41 41 74 74 74 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 41, Max flush: 74 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:54.989467091Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.034378327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.035354383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.035393627Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.035414564Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.035472336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349} {"msg":"Crucible dd7cc667-62b5-4a5f-9384-f4c9b9c1245d has session id: 7f62facd-f037-48b5-9887-d02e0f7fb061","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.035946441Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Upstairs opts: Upstairs UUID: dd7cc667-62b5-4a5f-9384-f4c9b9c1245d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.036006353Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Crucible stats registered with UUID: dd7cc667-62b5-4a5f-9384-f4c9b9c1245d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.036032559Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.067863002Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349} {"msg":"dd7cc667-62b5-4a5f-9384-f4c9b9c1245d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.067906376Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.067921884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.067944591Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.067959449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.067974507Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.067988125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.068001873Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.06802073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.068036448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.068061205Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_connection connected from Ok(127.0.0.1:44318)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.068185697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_connection connected from Ok(127.0.0.1:44472)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.068221193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_connection connected from Ok(127.0.0.1:50729)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.068245429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069117819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069144735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069160123Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069173991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069303643Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069343258Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069725885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.069803264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070085645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070371515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070401811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070417269Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070438116Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070452215Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070465742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070639649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070660416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[0]R flush_numbers[0..12]: [74, 69, 71, 72, 71, 63, 74, 67, 60, 60, 71, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070684142Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[0]R generation[0..12]: [41, 38, 39, 40, 39, 34, 41, 37, 33, 33, 39, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070698401Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070712568Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[1]R flush_numbers[0..12]: [74, 69, 71, 72, 71, 63, 74, 67, 60, 60, 71, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070726446Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[1]R generation[0..12]: [41, 38, 39, 40, 39, 34, 41, 37, 33, 33, 39, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070739875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070754063Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[2]R flush_numbers[0..12]: [74, 69, 71, 72, 71, 63, 74, 67, 60, 60, 71, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070767751Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[2]R generation[0..12]: [41, 38, 39, 40, 39, 34, 41, 37, 33, 33, 39, 36]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070785678Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070800426Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070813414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070827333Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070840361Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070853499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"":"downstairs","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070866927Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070879185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"0","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070900742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"1","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070914361Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"client":"2","":"downstairs","session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070928009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070940297Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"dd7cc667-62b5-4a5f-9384-f4c9b9c1245d is now active with session: 7f62facd-f037-48b5-9887-d02e0f7fb061","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070953195Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070965304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349,"session_id":"7f62facd-f037-48b5-9887-d02e0f7fb061"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:55.070977542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 527 len 36864 data: 2 5 5 6 5 5 5 5 5 02/30 Read block 380 len 12288 03/30 Write block 82 len 40960 data: 4 4 4 5 5 5 6 6 6 6 04/30 Read block 305 len 24576 05/30 Read block 509 len 40960 06/30 Read block 255 len 32768 07/30 Read block 17 len 40960 08/30 Write block 161 len 36864 data: 3 3 2 2 2 2 3 2 3 09/30 Read block 358 len 12288 10/30 Read block 554 len 40960 11/30 Read block 234 len 12288 12/30 Read block 175 len 24576 13/30 Write block 171 len 28672 data: 2 2 2 3 3 4 4 14/30 Read block 255 len 36864 15/30 Read block 270 len 24576 16/30 Read block 330 len 40960 17/30 Write block 256 len 16384 data: 3 3 4 5 18/30 Write block 221 len 12288 data: 6 5 4 19/30 Read block 53 len 40960 20/30 Read block 489 len 24576 21/30 Write block 402 len 36864 data: 4 5 6 7 7 8 9 9 7 22/30 Read block 497 len 16384 23/30 Write block 92 len 36864 data: 5 5 5 5 5 3 5 6 5 24/30 Write block 234 len 20480 data: 3 2 2 2 2 25/30 Read block 251 len 20480 26/30 Read block 498 len 4096 27/30 Write block 357 len 32768 data: 5 4 4 3 3 3 3 2 28/30 Read block 547 len 8192 29/30 Read block 188 len 4096 30/30 Read block 316 len 12288 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 9 Done Done Done false 2 Acked 1001 Read 3 Done Done Done false 3 Acked 1002 Write 10 Done Done Done false 4 Acked 1003 Read 6 Done Done Done false 5 Acked 1004 Read 10 Done Done Done false 6 Acked 1005 Read 8 Done Done Done false 7 Acked 1006 Read 10 Done Done Done false 8 Acked 1007 Write 9 Done Done Done false 9 Acked 1008 Read 3 Done Done Done false 10 Acked 1009 Read 10 Done Done Done false 11 Acked 1010 Read 3 Done Done Done false 12 Acked 1011 Read 6 Done Done Done false 13 Acked 1012 Write 7 Done Done Done false 14 Acked 1013 Read 9 Done Done Done false 15 Acked 1014 Read 6 Done Done Done false 16 Acked 1015 Read 10 Done Done Done false 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:55.093637452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1349} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 39 39 39 71 71 71 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 39 39 39 71 71 71 T T T 5 100-119 34 34 34 63 63 63 T T T 6 120-139 41 41 41 74 74 74 F F F 7 140-159 37 37 37 67 67 67 F F F 8 160-179 33 33 33 60 60 60 T T T 9 180-199 33 33 33 60 60 60 F F F 10 200-219 39 39 39 71 71 71 F F F 11 220-239 36 36 36 66 66 66 T T T 12 240-259 38 38 38 69 69 69 T T T 13 260-279 39 39 39 71 71 71 F F F 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 38 38 38 69 69 69 T T T 18 360-379 41 41 41 74 74 74 T T T 19 380-399 41 41 41 74 74 74 F F F 20 400-419 34 34 34 61 61 61 T T T 21 420-439 28 28 28 50 50 50 F F F 22 440-459 41 41 41 74 74 74 F F F 23 460-479 40 40 40 72 72 72 F F F 24 480-499 40 40 40 73 73 73 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 41 41 41 74 74 74 T T T 27 540-559 37 37 37 67 67 67 F F F 28 560-579 41 41 41 74 74 74 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 41, Max flush: 74 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 39 39 39 71 71 71 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 41 41 41 74 74 74 F F F 7 140-159 37 37 37 67 67 67 F F F 8 160-179 42 42 42 75 75 75 F F F 9 180-199 33 33 33 60 60 60 F F F 10 200-219 39 39 39 71 71 71 F F F 11 220-239 42 42 42 75 75 75 F F F 12 240-259 42 42 42 75 75 75 F F F 13 260-279 39 39 39 71 71 71 F F F 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 42 42 42 75 75 75 F F F 19 380-399 41 41 41 74 74 74 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 41 41 41 74 74 74 F F F 23 460-479 40 40 40 72 72 72 F F F 24 480-499 40 40 40 73 73 73 F F F 25 500-519 34 34 34 63 63 63 F F F 26 520-539 42 42 42 75 75 75 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 41 41 41 74 74 74 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 42, Max flush: 75 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.461464437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:59.505897759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.50639526Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.539752203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.540446337Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.540487451Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.540504669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.540522446Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357} {"msg":"Crucible 6969ea6a-38e7-4797-9aac-954145b52016 has session id: 3b9facaa-112b-4c28-b486-de4228edf5e9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.54093393Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Upstairs opts: Upstairs UUID: 6969ea6a-38e7-4797-9aac-954145b52016, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.540977803Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Crucible stats registered with UUID: 6969ea6a-38e7-4797-9aac-954145b52016","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.54100308Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57307621Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357} {"msg":"6969ea6a-38e7-4797-9aac-954145b52016 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573119893Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573151349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573197983Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573233238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573271383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573302418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573333344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57336375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573393756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573698224Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37646)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573737648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61188)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573772613Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54235)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.573800579Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.574308319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.574335616Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.574350234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.574364312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57437807Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.574403286Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575000854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57502865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575044088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575551608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575578204Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575592352Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575667812Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575686319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575699717Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575711866Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575724994Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[0]R flush_numbers[0..12]: [74, 69, 71, 72, 75, 75, 74, 67, 75, 60, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575738442Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[0]R generation[0..12]: [41, 38, 39, 40, 42, 42, 41, 37, 42, 33, 39, 42]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57575212Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575766278Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[1]R flush_numbers[0..12]: [74, 69, 71, 72, 75, 75, 74, 67, 75, 60, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575779996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[1]R generation[0..12]: [41, 38, 39, 40, 42, 42, 41, 37, 42, 33, 39, 42]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575798343Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575813481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[2]R flush_numbers[0..12]: [74, 69, 71, 72, 75, 75, 74, 67, 75, 60, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57582741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[2]R generation[0..12]: [41, 38, 39, 40, 42, 42, 41, 37, 42, 33, 39, 42]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575841018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575854926Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575867704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575880752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57589343Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575906149Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"":"downstairs","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-09-27T03:51:59.575919547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575938714Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"0","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575952232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"1","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575965471Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"client":"2","":"downstairs","session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.575985458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.576007894Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"6969ea6a-38e7-4797-9aac-954145b52016 is now active with session: 3b9facaa-112b-4c28-b486-de4228edf5e9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.576025322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.57603763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357,"session_id":"3b9facaa-112b-4c28-b486-de4228edf5e9"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:51:59.576050059Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 8192 data: 6 7 02/30 Read block 460 len 8192 03/30 Write block 513 len 16384 data: 4 5 6 5 04/30 Write block 569 len 28672 data: 6 5 5 5 5 4 3 05/30 Write block 459 len 12288 data: 6 5 5 06/30 Write block 409 len 28672 data: 10 8 6 4 4 4 3 07/30 Write block 192 len 40960 data: 2 2 2 2 4 4 4 3 3 3 08/30 Write block 179 len 32768 data: 3 3 5 5 6 5 4 4 09/30 Read block 398 len 24576 10/30 Write block 482 len 40960 data: 4 4 4 4 3 3 3 4 5 5 11/30 Write block 531 len 4096 data: 6 12/30 Read block 474 len 40960 13/30 Read block 415 len 40960 14/30 Write block 372 len 24576 data: 6 5 4 5 7 8 15/30 Read block 284 len 8192 16/30 Flush 17/30 Write block 382 len 12288 data: 4 3 3 18/30 Write block 267 len 24576 data: 4 5 4 4 5 5 19/30 Read block 529 len 32768 20/30 Write block 128 len 12288 data: 5 5 6 21/30 Read block 87 len 32768 22/30 Read block 379 len 24576 23/30 Write block 239 len 32768 data: 2 3 3 3 3 4 4 6 24/30 Read block 102 len 20480 25/30 Read block 61 len 40960 26/30 Read block 558 len 12288 27/30 Write block 394 len 32768 data: 4 4 4 3 2 3 3 3 28/30 Write block 447 len 20480 data: 3 3 3 3 3 29/30 Read block 390 len 32768 30/30 Read block 50 len 32768 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:51:59.598570978Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1357} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 44 44 44 76 76 76 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 41 41 41 74 74 74 T T T 7 140-159 37 37 37 67 67 67 F F F 8 160-179 44 44 44 76 76 76 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 42 42 42 75 75 75 T T T 12 240-259 42 42 42 75 75 75 T T T 13 260-279 39 39 39 71 71 71 T T T 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 41 41 41 74 74 74 T T T 20 400-419 44 44 44 76 76 76 T T T 21 420-439 28 28 28 50 50 50 F F F 22 440-459 44 44 44 76 76 76 T T T 23 460-479 44 44 44 76 76 76 F F F 24 480-499 44 44 44 76 76 76 F F F 25 500-519 44 44 44 76 76 76 F F F 26 520-539 44 44 44 76 76 76 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 44 44 44 76 76 76 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 44, Max flush: 76 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 44 44 44 76 76 76 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 45 45 45 77 77 77 F F F 7 140-159 37 37 37 67 67 67 F F F 8 160-179 44 44 44 76 76 76 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 45 45 45 77 77 77 F F F 12 240-259 45 45 45 77 77 77 F F F 13 260-279 45 45 45 77 77 77 F F F 14 280-299 37 37 37 67 67 67 F F F 15 300-319 37 37 37 67 67 67 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 45 45 45 77 77 77 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 44 44 44 76 76 76 F F F 24 480-499 44 44 44 76 76 76 F F F 25 500-519 44 44 44 76 76 76 F F F 26 520-539 44 44 44 76 76 76 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 44 44 44 76 76 76 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 45, Max flush: 77 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:03.974163079Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:04.017042446Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.01745158Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.049408885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.050088591Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.050125726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.050140524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.050154773Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365} {"msg":"Crucible 4318289f-2a10-4162-8bd7-b3fc60cbf427 has session id: 5c4fd121-abad-4cae-adbd-84adeefbf6ad","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.05060733Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Upstairs opts: Upstairs UUID: 4318289f-2a10-4162-8bd7-b3fc60cbf427, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.050646725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Crucible stats registered with UUID: 4318289f-2a10-4162-8bd7-b3fc60cbf427","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.050672021Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081679328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365} {"msg":"4318289f-2a10-4162-8bd7-b3fc60cbf427 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081708644Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081723722Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081745059Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081759057Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081778254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081792222Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081822178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081837116Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081851924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.081870421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:50261)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.082107679Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:42300)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.082161811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:64079)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.082201906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083006704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083033301Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083048449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083062497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083076275Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083089613Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083699659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083728085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.083743683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.08434121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084367336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084381825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084445796Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084473722Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.08448906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084507177Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084521235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[0]R flush_numbers[0..12]: [74, 69, 76, 72, 75, 75, 77, 67, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084583477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[0]R generation[0..12]: [41, 38, 44, 40, 42, 42, 45, 37, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084602654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084616992Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[1]R flush_numbers[0..12]: [74, 69, 76, 72, 75, 75, 77, 67, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.08463094Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[1]R generation[0..12]: [41, 38, 44, 40, 42, 42, 45, 37, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084649508Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084664955Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[2]R flush_numbers[0..12]: [74, 69, 76, 72, 75, 75, 77, 67, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084678924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[2]R generation[0..12]: [41, 38, 44, 40, 42, 42, 45, 37, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084692642Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.08470668Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084719888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084733096Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084746084Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084765992Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"":"downstairs","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.08477948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084791738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"0","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084805356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"1","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084818764Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"client":"2","":"downstairs","session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084832212Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084844551Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"4318289f-2a10-4162-8bd7-b3fc60cbf427 is now active with session: 5c4fd121-abad-4cae-adbd-84adeefbf6ad","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084860119Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:04.084872607Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365,"session_id":"5c4fd121-abad-4cae-adbd-84adeefbf6ad"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30region info: block size: 4096 bytes ,"time":" sub_volume 0 blocks / extent: 20 2024-09-27T03:52:04.084884975Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 121 len 32768 data: 7 7 5 5 5 6 6 6 02/30 Flush 03/30 Write block 285 len 20480 data: 5 5 6 5 2 04/30 Write block 365 len 16384 data: 2 2 3 5 05/30 Read block 361 len 28672 06/30 Read block 36 len 8192 07/30 Read block 207 len 12288 08/30 Read block 310 len 8192 09/30 Write block 523 len 24576 data: 5 5 5 4 3 6 10/30 Flush 11/30 Write block 177 len 12288 data: 5 3 4 12/30 Read block 18 len 12288 13/30 Write block 398 len 4096 data: 3 14/30 Flush 15/30 Write block 281 len 4096 data: 4 16/30 Write block 304 len 20480 data: 7 9 8 7 5 17/30 Read block 96 len 20480 18/30 Read block 55 len 32768 19/30 Write block 514 len 32768 data: 6 7 6 5 4 4 4 3 20/30 Read block 146 len 8192 21/30 Write block 244 len 4096 data: 5 22/30 Read block 144 len 8192 23/30 Read block 264 len 36864 24/30 Read block 497 len 32768 25/30 Write block 377 len 4096 data: 9 26/30 Read block 321 len 28672 27/30 Write block 306 len 28672 data: 9 8 6 4 5 4 4 28/30 Write block 140 len 28672 data: 8 8 6 6 5 6 6 29/30 Read block 39 len 4096 30/30 Read block 285 len 28672 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 15 Acked 1014 Write 1 Done Done Done false 16 Acked 1015 Write 5 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 16 16 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:04.108199995Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1365} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 44 44 44 76 76 76 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 46 46 46 78 78 78 F F F 7 140-159 37 37 37 67 67 67 T T T 8 160-179 46 46 46 80 80 80 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 45 45 45 77 77 77 F F F 12 240-259 45 45 45 77 77 77 T T T 13 260-279 45 45 45 77 77 77 F F F 14 280-299 46 46 46 79 79 79 T T T 15 300-319 37 37 37 67 67 67 T T T 16 320-339 25 25 25 44 44 44 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 46 46 46 79 79 79 T T T 19 380-399 46 46 46 80 80 80 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 44 44 44 76 76 76 F F F 24 480-499 44 44 44 76 76 76 F F F 25 500-519 44 44 44 76 76 76 T T T 26 520-539 46 46 46 79 79 79 T T T 27 540-559 37 37 37 67 67 67 F F F 28 560-579 44 44 44 76 76 76 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 46, Max flush: 80 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 44 44 44 76 76 76 F F F 3 060-079 40 40 40 72 72 72 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 46 46 46 78 78 78 F F F 7 140-159 47 47 47 81 81 81 F F F 8 160-179 46 46 46 80 80 80 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 45 45 45 77 77 77 F F F 12 240-259 47 47 47 81 81 81 F F F 13 260-279 45 45 45 77 77 77 F F F 14 280-299 47 47 47 81 81 81 F F F 15 300-319 47 47 47 81 81 81 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 44 44 44 76 76 76 F F F 24 480-499 44 44 44 76 76 76 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 47 47 47 81 81 81 F F F 27 540-559 37 37 37 67 67 67 F F F 28 560-579 44 44 44 76 76 76 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 47, Max flush: 81 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.479025645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.523631293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.524436672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.524493464Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.524513941Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.524532449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373} {"msg":"Crucible 08f562b3-421e-4d99-8841-719f272fe997 has session id: 22f3d05a-3cac-41d1-a039-66a158dff929","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.525024161Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Upstairs opts: Upstairs UUID: 08f562b3-421e-4d99-8841-719f272fe997, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.525060146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Crucible stats registered with UUID: 08f562b3-421e-4d99-8841-719f272fe997","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.525077894Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556365512Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373} {"msg":"08f562b3-421e-4d99-8841-719f272fe997 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556404136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556421794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.55645125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556473857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556505962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556537288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556576762Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556612078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556647053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556680598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_connection connected from Ok(127.0.0.1:60197)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556711374Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_connection connected from Ok(127.0.0.1:59983)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556743029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_connection connected from Ok(127.0.0.1:42827)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.556778455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.557591422Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.557618428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.557632997Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.557656013Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.55768272Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.557707356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558264459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558288836Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558417558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558854718Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558879694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558893852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558954974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.558994548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559013006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559026204Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559039962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[0]R flush_numbers[0..12]: [74, 69, 76, 72, 75, 75, 78, 81, 80, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.55905353Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[0]R generation[0..12]: [41, 38, 44, 40, 42, 42, 46, 47, 46, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559067518Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559081646Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[1]R flush_numbers[0..12]: [74, 69, 76, 72, 75, 75, 78, 81, 80, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559095084Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[1]R generation[0..12]: [41, 38, 44, 40, 42, 42, 46, 47, 46, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559108533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559122321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[2]R flush_numbers[0..12]: [74, 69, 76, 72, 75, 75, 78, 81, 80, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559139169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[2]R generation[0..12]: [41, 38, 44, 40, 42, 42, 46, 47, 46, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559153267Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559167425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559180313Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559193321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559206099Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559219037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"":"downstairs","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:52:08.559232305Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559249123Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"0","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559262411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"1","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.5592754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"client":"2","":"downstairs","session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559288658Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559301786Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"08f562b3-421e-4d99-8841-719f272fe997 is now active with session: 22f3d05a-3cac-41d1-a039-66a158dff929","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559314674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559326692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373,"session_id":"22f3d05a-3cac-41d1-a039-66a158dff929"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:08.559340821Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 536 len 36864 data: 5 3 3 3 6 7 6 6 7 02/30 Read block 59 len 28672 03/30 Read block 346 len 12288 04/30 Read block 427 len 20480 05/30 Write block 478 len 40960 data: 3 4 4 4 5 5 5 5 4 4 06/30 Read block 458 len 4096 07/30 Read block 366 len 28672 08/30 Read block 441 len 28672 09/30 Write block 160 len 36864 data: 3 4 4 3 3 3 3 4 3 10/30 Flush 11/30 Write block 221 len 24576 data: 7 6 5 7 6 6 12/30 Flush 13/30 Write block 558 len 36864 data: 3 4 4 4 3 4 5 6 6 14/30 Write block 531 len 24576 data: 7 6 6 6 6 6 15/30 Flush 16/30 Write block 416 len 20480 data: 3 4 4 4 4 17/30 Write block 530 len 40960 data: 7 8 7 7 7 7 7 4 4 4 18/30 Write block 403 len 40960 data: 6 7 8 8 9 10 11 9 7 5 19/30 Write block 408 len 28672 data: 11 12 10 8 6 5 5 20/30 Read block 8 len 28672 21/30 Flush 22/30 Write block 67 len 16384 data: 7 8 10 9 23/30 Read block 55 len 4096 24/30 Read block 585 len 40960 25/30 Flush 26/30 Write block 135 len 36864 data: 5 7 7 8 9 9 9 7 7 27/30 Read block 429 len 24576 28/30 Read block 69 len 16384 29/30 Read block 19 len 40960 30/30 Read block 463 len 24576 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:08.590416998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1373} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 44 44 44 76 76 76 F F F 3 060-079 48 48 48 86 86 86 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 46 46 46 78 78 78 T T T 7 140-159 47 47 47 81 81 81 T T T 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 48 48 48 83 83 83 F F F 12 240-259 47 47 47 81 81 81 F F F 13 260-279 45 45 45 77 77 77 F F F 14 280-299 47 47 47 81 81 81 F F F 15 300-319 47 47 47 81 81 81 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 48 48 48 85 85 85 F F F 21 420-439 48 48 48 85 85 85 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 48 48 48 82 82 82 F F F 24 480-499 48 48 48 82 82 82 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 48 48 48 85 85 85 F F F 27 540-559 48 48 48 84 84 84 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 48, Max flush: 86 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 38 38 38 69 69 69 F F F 2 040-059 44 44 44 76 76 76 F F F 3 060-079 48 48 48 86 86 86 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 49 49 49 87 87 87 F F F 7 140-159 49 49 49 87 87 87 F F F 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 48 48 48 83 83 83 F F F 12 240-259 47 47 47 81 81 81 F F F 13 260-279 45 45 45 77 77 77 F F F 14 280-299 47 47 47 81 81 81 F F F 15 300-319 47 47 47 81 81 81 F F F 16 320-339 25 25 25 44 44 44 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 48 48 48 85 85 85 F F F 21 420-439 48 48 48 85 85 85 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 48 48 48 82 82 82 F F F 24 480-499 48 48 48 82 82 82 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 48 48 48 85 85 85 F F F 27 540-559 48 48 48 84 84 84 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 49, Max flush: 87 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.871222553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.915921369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.916667766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.91670365Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.916718308Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.916732526Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381} {"msg":"Crucible c4cb69ed-af6f-43b1-a1f3-b11ab3e22391 has session id: 6c4831e8-421f-4eeb-aa3d-99d38496e8ad","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.917150669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Upstairs opts: Upstairs UUID: c4cb69ed-af6f-43b1-a1f3-b11ab3e22391, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.917189793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Crucible stats registered with UUID: c4cb69ed-af6f-43b1-a1f3-b11ab3e22391","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.91721406Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.948957595Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381} {"msg":"c4cb69ed-af6f-43b1-a1f3-b11ab3e22391 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949029695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949048263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949077418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949094486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949111124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949127182Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.9491437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949160107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949186243Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.94921368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:36866)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949328434Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:65338)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949378907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:45403)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.949413772Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.950282372Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.950322377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.950352962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.950374689Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.95044261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.950497762Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.950976496Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951102169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951258587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951566125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951591421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951605949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951645954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951666351Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951680239Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951738901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951757288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[0]R flush_numbers[0..12]: [74, 69, 76, 86, 75, 75, 87, 87, 82, 76, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951780055Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[0]R generation[0..12]: [41, 38, 44, 48, 42, 42, 49, 49, 48, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951793963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951808151Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[1]R flush_numbers[0..12]: [74, 69, 76, 86, 75, 75, 87, 87, 82, 76, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.95182185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[1]R generation[0..12]: [41, 38, 44, 48, 42, 42, 49, 49, 48, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951835548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951849616Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[2]R flush_numbers[0..12]: [74, 69, 76, 86, 75, 75, 87, 87, 82, 76, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951863184Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[2]R generation[0..12]: [41, 38, 44, 48, 42, 42, 49, 49, 48, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951876602Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.95189029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951903198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951919256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951932384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951945193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"":"downstairs","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951958501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951970679Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"0"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.951991626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"1","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.952004694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"client":"2","":"downstairs","session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.952017792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.952030011Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"c4cb69ed-af6f-43b1-a1f3-b11ab3e22391 is now active with session: 6c4831e8-421f-4eeb-aa3d-99d38496e8ad","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.952043079Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.952055147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381,"session_id":"6c4831e8-421f-4eeb-aa3d-99d38496e8ad"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:12.952067385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 16384 data: 5 5 4 5 02/30 Flush 03/30 Write block 33 len 32768 data: 6 5 4 5 6 6 5 6 04/30 Read block 35 len 12288 05/30 Flush 06/30 Write block 349 len 36864 data: 9 7 9 9 8 5 6 5 6 07/30 Read block 44 len 8192 08/30 Write block 218 len 20480 data: 4 4 4 8 7 09/30 Read block 501 len 24576 10/30 Write block 311 len 12288 data: 5 5 4 11/30 Write block 413 len 24576 data: 6 6 4 4 5 5 12/30 Write block 54 len 16384 data: 2 2 3 3 13/30 Read block 457 len 12288 14/30 Read block 189 len 8192 15/30 Read block 534 len 40960 16/30 Read block 168 len 32768 17/30 Write block 472 len 40960 data: 3 3 4 4 3 3 4 5 5 5 18/30 Read block 584 len 16384 19/30 Write block 326 len 40960 data: 5 4 2 2 2 2 2 3 3 3 20/30 Read block 214 len 36864 21/30 Read block 254 len 20480 22/30 Write block 52 len 28672 data: 4 4 3 3 4 4 3 23/30 Read block 182 len 20480 24/30 Flush 25/30 Write block 334 len 20480 data: 4 4 3 4 4 26/30 Write block 296 len 40960 data: 5 4 5 4 4 6 7 6 8 10 27/30 Read block 348 len 32768 28/30 Read block 173 len 40960 29/30 Read block 34 len 20480 30/30 Write block 354 len 16384 data: 6 7 6 7 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:12.980541852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1381} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 50 50 50 89 89 89 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 48 48 48 86 86 86 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 49 49 49 87 87 87 F F F 7 140-159 49 49 49 87 87 87 F F F 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 50 50 50 90 90 90 F F F 11 220-239 50 50 50 90 90 90 F F F 12 240-259 47 47 47 81 81 81 F F F 13 260-279 45 45 45 77 77 77 F F F 14 280-299 47 47 47 81 81 81 T T T 15 300-319 50 50 50 90 90 90 T T T 16 320-339 50 50 50 90 90 90 T T T 17 340-359 50 50 50 90 90 90 T T T 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 48 48 48 85 85 85 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 50 50 50 90 90 90 F F F 24 480-499 50 50 50 90 90 90 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 48 48 48 85 85 85 F F F 27 540-559 48 48 48 84 84 84 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 50, Max flush: 90 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 50 50 50 89 89 89 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 48 48 48 86 86 86 F F F 4 080-099 42 42 42 75 75 75 F F F 5 100-119 42 42 42 75 75 75 F F F 6 120-139 49 49 49 87 87 87 F F F 7 140-159 49 49 49 87 87 87 F F F 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 50 50 50 90 90 90 F F F 11 220-239 50 50 50 90 90 90 F F F 12 240-259 47 47 47 81 81 81 F F F 13 260-279 45 45 45 77 77 77 F F F 14 280-299 51 51 51 91 91 91 F F F 15 300-319 51 51 51 91 91 91 F F F 16 320-339 51 51 51 91 91 91 F F F 17 340-359 51 51 51 91 91 91 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 48 48 48 85 85 85 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 50 50 50 90 90 90 F F F 24 480-499 50 50 50 90 90 90 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 48 48 48 85 85 85 F F F 27 540-559 48 48 48 84 84 84 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 51, Max flush: 91 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.283705529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.328307088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.329035778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.329085031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.329109127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.329132844Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389} {"msg":"Crucible cd95cf3b-cdf3-4583-a874-664b06f99252 has session id: de066e7d-49a0-4e45-9f04-1388c685c8d7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.329608598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Upstairs opts: Upstairs UUID: cd95cf3b-cdf3-4583-a874-664b06f99252, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.329645633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Crucible stats registered with UUID: cd95cf3b-cdf3-4583-a874-664b06f99252","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.329662031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361457939Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389} {"msg":"cd95cf3b-cdf3-4583-a874-664b06f99252 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361488745Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361503753Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.36152513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361540278Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361553846Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361567544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361580922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361635514Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361653282Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.36174325Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:52836)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.361827688Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56658)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362056656Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58450)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362114259Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362629357Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362657284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362677321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362692748Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362706577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.362766128Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.36325821Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.363370545Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.363514535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.363974062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364001468Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364016166Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364028624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364042722Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.36405625Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364068709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364082047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[0]R flush_numbers[0..12]: [74, 89, 90, 86, 75, 75, 87, 87, 82, 76, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364105693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[0]R generation[0..12]: [41, 50, 50, 48, 42, 42, 49, 49, 48, 44, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364120191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.36413428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[1]R flush_numbers[0..12]: [74, 89, 90, 86, 75, 75, 87, 87, 82, 76, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364147858Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[1]R generation[0..12]: [41, 50, 50, 48, 42, 42, 49, 49, 48, 44, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364161436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364175334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[2]R flush_numbers[0..12]: [74, 89, 90, 86, 75, 75, 87, 87, 82, 76, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364188762Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[2]R generation[0..12]: [41, 50, 50, 48, 42, 42, 49, 49, 48, 44, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.36420239Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364216248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364229176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364242345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364255053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364267891Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"":"downstairs","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364281249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:52:17.364297247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"0","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364316654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"1","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364330072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"client":"2","":"downstairs","session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364343691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364355959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"cd95cf3b-cdf3-4583-a874-664b06f99252 is now active with session: de066e7d-49a0-4e45-9f04-1388c685c8d7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364368867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364380725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389,"session_id":"de066e7d-49a0-4e45-9f04-1388c685c8d7"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:17.364393094Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 24576 data: 4 3 3 3 3 3 02/30 Read block 319 len 40960 03/30 Flush 04/30 Write block 130 len 28672 data: 7 6 6 6 6 6 8 05/30 Write block 96 len 20480 data: 6 4 6 7 6 06/30 Write block 294 len 24576 data: 5 5 6 5 6 5 07/30 Write block 228 len 24576 data: 5 5 3 2 2 2 08/30 Flush 09/30 Write block 292 len 24576 data: 6 6 6 6 7 6 10/30 Read block 554 len 28672 11/30 Read block 206 len 4096 12/30 Write block 425 len 12288 data: 4 3 3 13/30 Write block 143 len 28672 data: 8 6 7 7 6 6 6 14/30 Write block 136 len 40960 data: 9 8 9 10 10 10 8 9 7 8 15/30 Read block 443 len 20480 16/30 Read block 119 len 24576 17/30 Write block 39 len 4096 data: 6 18/30 Write block 273 len 36864 data: 4 4 4 4 4 4 5 4 5 19/30 Read block 208 len 4096 20/30 Write block 315 len 12288 data: 3 3 5 21/30 Read block 534 len 28672 22/30 Read block 520 len 24576 23/30 Read block 584 len 36864 24/30 Flush 25/30 Write block 532 len 12288 data: 8 8 8 26/30 Write block 330 len 12288 data: 3 3 3 27/30 Read block 497 len 16384 28/30 Write block 64 len 16384 data: 4 5 5 8 29/30 Read block 390 len 16384 30/30 Write block 286 len 32768 data: 6 7 6 3 4 6 7 7 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:17.390398851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1389} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 48 48 48 86 86 86 T T T 4 080-099 52 52 52 93 93 93 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 52 52 52 94 94 94 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 50 50 50 90 90 90 F F F 11 220-239 52 52 52 93 93 93 F F F 12 240-259 47 47 47 81 81 81 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 52 52 52 94 94 94 T T T 15 300-319 52 52 52 94 94 94 F F F 16 320-339 51 51 51 91 91 91 T T T 17 340-359 51 51 51 91 91 91 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 52 52 52 94 94 94 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 50 50 50 90 90 90 F F F 24 480-499 50 50 50 90 90 90 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 48 48 48 85 85 85 T T T 27 540-559 48 48 48 84 84 84 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 52, Max flush: 94 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 74 74 74 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 53 53 53 95 95 95 F F F 4 080-099 52 52 52 93 93 93 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 52 52 52 94 94 94 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 50 50 50 90 90 90 F F F 11 220-239 52 52 52 93 93 93 F F F 12 240-259 47 47 47 81 81 81 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 53 53 53 95 95 95 F F F 15 300-319 52 52 52 94 94 94 F F F 16 320-339 53 53 53 95 95 95 F F F 17 340-359 51 51 51 91 91 91 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 52 52 52 94 94 94 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 50 50 50 90 90 90 F F F 24 480-499 50 50 50 90 90 90 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 53 53 53 95 95 95 F F F 27 540-559 48 48 48 84 84 84 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 41 41 41 74 74 74 F F F Max gen: 53, Max flush: 95 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.698939345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.744719151Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.745503543Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.745558895Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.745578813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.7455973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397} {"msg":"Crucible a6a5306a-0faa-4941-8fff-6a637cbb0f9c has session id: d1c81f24-b359-45df-99f3-d51a53e2c135","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.746005054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Upstairs opts: Upstairs UUID: a6a5306a-0faa-4941-8fff-6a637cbb0f9c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.746056307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Crucible stats registered with UUID: a6a5306a-0faa-4941-8fff-6a637cbb0f9c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.746086372Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777483645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397} {"msg":"a6a5306a-0faa-4941-8fff-6a637cbb0f9c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.77752153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777540137Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777571813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.77759581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777612987Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777629405Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777645823Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.77766199Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777678878Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777696176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_connection connected from Ok(127.0.0.1:37051)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777839016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_connection connected from Ok(127.0.0.1:47008)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777887789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_connection connected from Ok(127.0.0.1:35799)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.777926784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.778713865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.778742071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.778757359Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.778771127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.778793124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.77882071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.77933185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.779431836Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.779491078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.779935396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.779968382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.77998483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.78005749Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780085976Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780103983Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780116891Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780130829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[0]R flush_numbers[0..12]: [74, 94, 90, 95, 93, 93, 94, 94, 82, 76, 90, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780144738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[0]R generation[0..12]: [41, 52, 50, 53, 52, 52, 52, 52, 48, 44, 50, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780165775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780189132Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[1]R flush_numbers[0..12]: [74, 94, 90, 95, 93, 93, 94, 94, 82, 76, 90, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780210558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[1]R generation[0..12]: [41, 52, 50, 53, 52, 52, 52, 52, 48, 44, 50, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780230146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780249543Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[2]R flush_numbers[0..12]: [74, 94, 90, 95, 93, 93, 94, 94, 82, 76, 90, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.7802726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[2]R generation[0..12]: [41, 52, 50, 53, 52, 52, 52, 52, 48, 44, 50, 52]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780295727Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780321313Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.7803417Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780355989Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Next flush: 96","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2024-09-27T03:52:21.780400532Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.78041677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"":"downstairs","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780430808Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780443136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"0","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780456435Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"1","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780469533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"client":"2","":"downstairs","session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780482871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780498489Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"a6a5306a-0faa-4941-8fff-6a637cbb0f9c is now active with session: d1c81f24-b359-45df-99f3-d51a53e2c135","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780511567Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780523525Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397,"session_id":"d1c81f24-b359-45df-99f3-d51a53e2c135"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:21.780535604Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 148 len 24576 data: 7 7 4 4 4 5 03/30 Read block 348 len 8192 04/30 Read block 37 len 40960 05/30 Read block 581 len 36864 06/30 Write block 41 len 4096 data: 7 07/30 Read block 54 len 8192 08/30 Write block 67 len 28672 data: 9 9 11 10 10 10 10 09/30 Write block 8 len 36864 data: 5 5 7 5 5 6 6 6 6 10/30 Read block 128 len 32768 11/30 Read block 555 len 24576 12/30 Write block 583 len 16384 data: 5 5 4 4 13/30 Write block 455 len 28672 data: 7 6 8 6 7 6 6 14/30 Read block 344 len 24576 15/30 Write block 299 len 12288 data: 6 5 7 16/30 Read block 511 len 16384 17/30 Write block 80 len 12288 data: 6 4 5 18/30 Write block 227 len 24576 data: 6 6 6 4 3 3 19/30 Write block 554 len 24576 data: 4 3 4 3 4 5 20/30 Write block 238 len 40960 data: 4 4 4 4 4 4 6 5 7 7 21/30 Read block 530 len 40960 22/30 Flush 23/30 Write block 426 len 24576 data: 4 4 2 2 2 2 24/30 Flush 25/30 Write block 329 len 8192 data: 3 4 26/30 Read block 75 len 20480 27/30 Read block 337 len 32768 28/30 Write block 205 len 4096 data: 3 29/30 Write block 519 len 16384 data: 5 5 4 4 30/30 Write block 293 len 8192 data: 8 7 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 4 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:21.802290109Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1397} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 97 97 97 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 54 54 54 97 97 97 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 52 52 52 94 94 94 F F F 7 140-159 54 54 54 97 97 97 F F F 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 50 50 50 90 90 90 T T T 11 220-239 54 54 54 97 97 97 F F F 12 240-259 54 54 54 97 97 97 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 97 97 97 T T T 15 300-319 54 54 54 97 97 97 F F F 16 320-339 53 53 53 95 95 95 T T T 17 340-359 51 51 51 91 91 91 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 54 54 54 98 98 98 F F F 22 440-459 54 54 54 97 97 97 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 50 50 50 90 90 90 F F F 25 500-519 47 47 47 81 81 81 T T T 26 520-539 53 53 53 95 95 95 T T T 27 540-559 54 54 54 97 97 97 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 54 54 54 97 97 97 F F F Max gen: 54, Max flush: 98 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 97 97 97 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 54 54 54 97 97 97 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 52 52 52 94 94 94 F F F 7 140-159 54 54 54 97 97 97 F F F 8 160-179 48 48 48 82 82 82 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 55 55 55 99 99 99 F F F 11 220-239 54 54 54 97 97 97 F F F 12 240-259 54 54 54 97 97 97 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 55 55 55 99 99 99 F F F 15 300-319 54 54 54 97 97 97 F F F 16 320-339 55 55 55 99 99 99 F F F 17 340-359 51 51 51 91 91 91 F F F 18 360-379 47 47 47 81 81 81 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 54 54 54 98 98 98 F F F 22 440-459 54 54 54 97 97 97 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 50 50 50 90 90 90 F F F 25 500-519 55 55 55 99 99 99 F F F 26 520-539 55 55 55 99 99 99 F F F 27 540-559 54 54 54 97 97 97 F F F 28 560-579 48 48 48 84 84 84 F F F 29 580-599 54 54 54 97 97 97 F F F Max gen: 55, Max flush: 99 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.102541418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.146567367Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.147186581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.147221506Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.147240404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.147255462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405} {"msg":"Crucible 03fb6886-06d3-4be0-a84b-bfe941892e50 has session id: 6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.147683253Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Upstairs opts: Upstairs UUID: 03fb6886-06d3-4be0-a84b-bfe941892e50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.147721337Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Crucible stats registered with UUID: 03fb6886-06d3-4be0-a84b-bfe941892e50","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.147744134Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178652915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405} {"msg":"03fb6886-06d3-4be0-a84b-bfe941892e50 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.17868502Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178702938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178731124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178748481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178765829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178792055Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178826121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178863495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178897691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.178930686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_connection connected from Ok(127.0.0.1:42301)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179015385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_connection connected from Ok(127.0.0.1:36848)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179060648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_connection connected from Ok(127.0.0.1:52613)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179091364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179854498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179882275Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179899562Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179927528Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179953505Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.179979411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.180560691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.180588687Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.180603555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181178086Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181204802Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.18121896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181232628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181246266Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181259404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181271533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181284751Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[0]R flush_numbers[0..12]: [97, 94, 97, 97, 97, 93, 94, 97, 82, 76, 99, 97]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181298429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[0]R generation[0..12]: [54, 52, 54, 54, 54, 52, 52, 54, 48, 44, 55, 54]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181312327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181326245Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[1]R flush_numbers[0..12]: [97, 94, 97, 97, 97, 93, 94, 97, 82, 76, 99, 97]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181339804Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[1]R generation[0..12]: [54, 52, 54, 54, 54, 52, 52, 54, 48, 44, 55, 54]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181353091Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.18136709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[2]R flush_numbers[0..12]: [97, 94, 97, 97, 97, 93, 94, 97, 82, 76, 99, 97]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181380518Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[2]R generation[0..12]: [54, 52, 54, 54, 54, 52, 52, 54, 48, 44, 55, 54]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181398875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181413683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181426841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181440119Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:52:26.181453098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181475285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"":"downstairs","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181488273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181500151Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"0","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181513389Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"1","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181526428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"client":"2","":"downstairs","session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181539596Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181551934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"03fb6886-06d3-4be0-a84b-bfe941892e50 is now active with session: 6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181564752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181576881Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405,"session_id":"6fedc8e1-a91e-474b-9bd1-c6c3f8fbc118"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:26.181588909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 246 len 20480 data: 8 8 7 8 7 02/30 Read block 77 len 4096 03/30 Flush 04/30 Write block 158 len 40960 data: 3 3 4 5 5 4 4 4 4 5 05/30 Write block 125 len 12288 data: 6 7 7 06/30 Write block 579 len 4096 data: 4 07/30 Read block 152 len 40960 08/30 Read block 337 len 16384 09/30 Write block 226 len 20480 data: 7 7 7 7 5 10/30 Read block 503 len 24576 11/30 Read block 511 len 20480 12/30 Write block 285 len 24576 data: 6 7 8 7 4 5 13/30 Read block 205 len 8192 14/30 Read block 203 len 8192 15/30 Flush 16/30 Write block 432 len 4096 data: 3 17/30 Write block 323 len 28672 data: 4 4 5 6 5 3 4 18/30 Read block 89 len 4096 19/30 Read block 483 len 20480 20/30 Write block 364 len 24576 data: 3 3 3 4 6 7 21/30 Read block 565 len 24576 22/30 Flush 23/30 Write block 548 len 20480 data: 6 4 2 2 2 24/30 Write block 362 len 24576 data: 4 4 4 4 4 5 25/30 Write block 368 len 12288 data: 7 8 5 26/30 Read block 114 len 12288 27/30 Read block 447 len 8192 28/30 Write block 483 len 16384 data: 6 6 6 5 29/30 Write block 496 len 32768 data: 4 6 5 6 7 7 8 7 30/30 Write block 85 len 28672 data: 6 6 6 7 7 7 7 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:26.203971147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1405} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 97 97 97 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 54 54 54 97 97 97 T T T 5 100-119 52 52 52 93 93 93 F F F 6 120-139 56 56 56 101 101 101 F F F 7 140-159 56 56 56 101 101 101 F F F 8 160-179 56 56 56 101 101 101 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 55 55 55 99 99 99 F F F 11 220-239 56 56 56 101 101 101 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 56 56 56 101 101 101 F F F 15 300-319 54 54 54 97 97 97 F F F 16 320-339 56 56 56 102 102 102 F F F 17 340-359 51 51 51 91 91 91 F F F 18 360-379 56 56 56 102 102 102 T T T 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 56 56 56 102 102 102 F F F 22 440-459 54 54 54 97 97 97 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 50 50 50 90 90 90 T T T 25 500-519 55 55 55 99 99 99 T T T 26 520-539 55 55 55 99 99 99 F F F 27 540-559 54 54 54 97 97 97 T T T 28 560-579 56 56 56 101 101 101 F F F 29 580-599 54 54 54 97 97 97 F F F Max gen: 56, Max flush: 102 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 97 97 97 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 56 56 56 101 101 101 F F F 7 140-159 56 56 56 101 101 101 F F F 8 160-179 56 56 56 101 101 101 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 55 55 55 99 99 99 F F F 11 220-239 56 56 56 101 101 101 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 56 56 56 101 101 101 F F F 15 300-319 54 54 54 97 97 97 F F F 16 320-339 56 56 56 102 102 102 F F F 17 340-359 51 51 51 91 91 91 F F F 18 360-379 57 57 57 103 103 103 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 F F F 21 420-439 56 56 56 102 102 102 F F F 22 440-459 54 54 54 97 97 97 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 57 57 57 103 103 103 F F F 25 500-519 57 57 57 103 103 103 F F F 26 520-539 55 55 55 99 99 99 F F F 27 540-559 57 57 57 103 103 103 F F F 28 560-579 56 56 56 101 101 101 F F F 29 580-599 54 54 54 97 97 97 F F F Max gen: 57, Max flush: 103 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.508851477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.555172789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.555829848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.555864303Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.555877871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.555892089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413} {"msg":"Crucible 77c1cd2b-a593-4c6e-b14d-d1fd7e9a57c2 has session id: 742ea753-85b9-45f3-af6d-2706288f57c3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.556312221Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Upstairs opts: Upstairs UUID: 77c1cd2b-a593-4c6e-b14d-d1fd7e9a57c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.556353555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Crucible stats registered with UUID: 77c1cd2b-a593-4c6e-b14d-d1fd7e9a57c2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.556377122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587259506Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413} {"msg":"77c1cd2b-a593-4c6e-b14d-d1fd7e9a57c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587289512Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.58730718Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587334286Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587351574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587368181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587384639Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587401417Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587427093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587459869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587491414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38847)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587528719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54496)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587568524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55412)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.587602429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.588505734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.588532221Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.588547048Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.588561137Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.58867832Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.588713225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589177801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589209077Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589444675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589871136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589899712Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.58991553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589928688Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589946225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589960683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.589973131Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.58998659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[0]R flush_numbers[0..12]: [97, 94, 97, 97, 103, 93, 101, 101, 101, 76, 99, 101]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590000488Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[0]R generation[0..12]: [54, 52, 54, 54, 57, 52, 56, 56, 56, 44, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590014426Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590028524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[1]R flush_numbers[0..12]: [97, 94, 97, 97, 103, 93, 101, 101, 101, 76, 99, 101]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590042112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[1]R generation[0..12]: [54, 52, 54, 54, 57, 52, 56, 56, 56, 44, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.59005573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590069758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[2]R flush_numbers[0..12]: [97, 94, 97, 97, 103, 93, 101, 101, 101, 76, 99, 101]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590083536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[2]R generation[0..12]: [54, 52, 54, 54, 57, 52, 56, 56, 56, 44, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590097304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590111112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590124051Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590137229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590152247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":Wait for a query_work_queue command to finish before sending IO 1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590171914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"":"downstairs","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590185232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.59019736Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"0","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590210629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"1","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590224037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"client":"2","":"downstairs","session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590237595Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590249883Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"77c1cd2b-a593-4c6e-b14d-d1fd7e9a57c2 is now active with session: 742ea753-85b9-45f3-af6d-2706288f57c3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590262722Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.59027553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413,"session_id":"742ea753-85b9-45f3-af6d-2706288f57c3"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:30.590287788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 556 len 24576 data: 5 4 5 6 5 5 02/30 Read block 244 len 32768 03/30 Read block 275 len 40960 04/30 Write block 304 len 20480 data: 9 11 10 9 7 05/30 Write block 6 len 20480 data: 4 4 6 6 8 06/30 Read block 276 len 4096 07/30 Write block 451 len 12288 data: 4 4 5 08/30 Write block 147 len 4096 data: 7 09/30 Read block 48 len 40960 10/30 Read block 336 len 28672 11/30 Read block 52 len 4096 12/30 Flush 13/30 Write block 597 len 12288 data: 3 3 2 14/30 Read block 529 len 28672 15/30 Flush 16/30 Flush 17/30 Write block 208 len 40960 data: 4 3 4 4 4 5 5 4 5 5 18/30 Write block 368 len 24576 data: 8 9 6 5 7 6 19/30 Write block 145 len 12288 data: 9 8 8 20/30 Write block 231 len 28672 data: 4 4 3 5 4 4 4 21/30 Read block 271 len 24576 22/30 Read block 419 len 32768 23/30 Read block 467 len 20480 24/30 Read block 164 len 4096 25/30 Flush 26/30 Write block 346 len 36864 data: 8 9 10 10 8 10 10 9 7 27/30 Read block 370 len 24576 28/30 Read block 321 len 8192 29/30 Write block 374 len 24576 data: 5 6 8 10 6 7 30/30 Write block 411 len 40960 data: 9 7 7 7 5 5 6 6 5 5 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:30.615183819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1413} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 104 104 104 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 56 56 56 101 101 101 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 56 56 56 101 101 101 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 58 58 58 107 107 107 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 56 56 56 101 101 101 F F F 15 300-319 58 58 58 104 104 104 F F F 16 320-339 56 56 56 102 102 102 F F F 17 340-359 51 51 51 91 91 91 T T T 18 360-379 58 58 58 107 107 107 T T T 19 380-399 46 46 46 80 80 80 F F F 20 400-419 50 50 50 90 90 90 T T T 21 420-439 56 56 56 102 102 102 T T T 22 440-459 58 58 58 104 104 104 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 57 57 57 103 103 103 F F F 25 500-519 57 57 57 103 103 103 F F F 26 520-539 55 55 55 99 99 99 F F F 27 540-559 58 58 58 104 104 104 F F F 28 560-579 58 58 58 104 104 104 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 58, Max flush: 107 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 104 104 104 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 56 56 56 101 101 101 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 56 56 56 101 101 101 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 58 58 58 107 107 107 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 56 56 56 101 101 101 F F F 15 300-319 58 58 58 104 104 104 F F F 16 320-339 56 56 56 102 102 102 F F F 17 340-359 59 59 59 108 108 108 F F F 18 360-379 59 59 59 108 108 108 F F F 19 380-399 46 46 46 80 80 80 F F F 20 400-419 59 59 59 108 108 108 F F F 21 420-439 59 59 59 108 108 108 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 57 57 57 103 103 103 F F F 25 500-519 57 57 57 103 103 103 F F F 26 520-539 55 55 55 99 99 99 F F F 27 540-559 58 58 58 104 104 104 F F F 28 560-579 58 58 58 104 104 104 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 59, Max flush: 108 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.911847913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.955967639Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.956597242Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.956633857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.956648425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.956669882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421} {"msg":"Crucible 2da1e62a-2554-4736-89b5-f316b42a5187 has session id: f39a8444-6d2b-49a5-b83b-58d389a31050","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.957142817Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Upstairs opts: Upstairs UUID: 2da1e62a-2554-4736-89b5-f316b42a5187, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.957193869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Crucible stats registered with UUID: 2da1e62a-2554-4736-89b5-f316b42a5187","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.957218416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988299383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421} {"msg":"2da1e62a-2554-4736-89b5-f316b42a5187 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988338607Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988364424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988410577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988447822Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988482218Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988514683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988547769Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988572985Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988591143Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.98860842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_connection connected from Ok(127.0.0.1:62503)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988758089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_connection connected from Ok(127.0.0.1:56012)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988800064Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_connection connected from Ok(127.0.0.1:48232)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.988844968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.989599913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.989628699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.989647317Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.989661515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.989674923Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.989690991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.990312015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.99035034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.990376866Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.990906383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.990933169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.990947527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991024767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991044854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991058092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.99107017Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991083438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[0]R flush_numbers[0..12]: [104, 94, 97, 97, 103, 93, 101, 107, 101, 76, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991100606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[0]R generation[0..12]: [58, 52, 54, 54, 57, 52, 56, 58, 56, 44, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991114614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991128692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[1]R flush_numbers[0..12]: [104, 94, 97, 97, 103, 93, 101, 107, 101, 76, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.99114239Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[1]R generation[0..12]: [58, 52, 54, 54, 57, 52, 56, 58, 56, 44, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991156318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991170036Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[2]R flush_numbers[0..12]: [104, 94, 97, 97, 103, 93, 101, 107, 101, 76, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991183585Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[2]R generation[0..12]: [58, 52, 54, 54, 57, 52, 56, 58, 56, 44, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991197223Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991210971Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991223769Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991236917Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991249785Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991262563Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"":"downstairs","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991275602Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.99128762Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"0","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991303708Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA"Wait for a query_work_queue command to finish before sending IO ,"pid":1421,"client":"1","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991325125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"client":"2","":"downstairs","session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991338423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991350711Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"2da1e62a-2554-4736-89b5-f316b42a5187 is now active with session: f39a8444-6d2b-49a5-b83b-58d389a31050","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.99136361Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991375808Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421,"session_id":"f39a8444-6d2b-49a5-b83b-58d389a31050"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:34.991388026Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 358 len 24576 data: 5 5 4 4 5 5 02/30 Read block 336 len 16384 03/30 Write block 129 len 16384 data: 6 8 7 7 04/30 Read block 326 len 16384 05/30 Read block 235 len 12288 06/30 Flush 07/30 Write block 329 len 20480 data: 5 5 4 4 4 08/30 Read block 34 len 32768 09/30 Write block 128 len 28672 data: 7 7 9 8 8 7 7 10/30 Read block 241 len 24576 11/30 Flush 12/30 Write block 354 len 4096 data: 8 13/30 Write block 515 len 36864 data: 8 7 6 5 6 6 5 5 6 14/30 Write block 386 len 4096 data: 2 15/30 Read block 295 len 12288 16/30 Read block 505 len 4096 17/30 Read block 46 len 4096 18/30 Read block 536 len 32768 19/30 Read block 415 len 40960 20/30 Read block 46 len 24576 21/30 Write block 177 len 40960 data: 6 4 5 4 6 6 7 6 5 5 22/30 Read block 567 len 16384 23/30 Read block 554 len 12288 24/30 Read block 91 len 12288 25/30 Read block 527 len 36864 26/30 Write block 421 len 4096 data: 4 27/30 Write block 523 len 20480 data: 7 6 6 5 4 28/30 Write block 542 len 20480 data: 7 7 8 7 8 29/30 Read block 502 len 32768 30/30 Read block 99 len 12288 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 12 Acked 1011 Write 1 Done Done Done false 13 Acked 1012 Write 9 Done Done Done false 14 Acked 1013 Write 1 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 1 Done Done Done false 17 Acked 1016 Read 1 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 10 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 19 19 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:35.014984867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1421} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 104 104 104 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 60 60 60 110 110 110 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 56 56 56 101 101 101 T T T 9 180-199 44 44 44 76 76 76 T T T 10 200-219 58 58 58 107 107 107 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 56 56 56 101 101 101 F F F 15 300-319 58 58 58 104 104 104 F F F 16 320-339 60 60 60 110 110 110 F F F 17 340-359 60 60 60 109 109 109 T T T 18 360-379 60 60 60 109 109 109 F F F 19 380-399 46 46 46 80 80 80 T T T 20 400-419 59 59 59 108 108 108 F F F 21 420-439 59 59 59 108 108 108 T T T 22 440-459 58 58 58 104 104 104 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 57 57 57 103 103 103 F F F 25 500-519 57 57 57 103 103 103 T T T 26 520-539 55 55 55 99 99 99 T T T 27 540-559 58 58 58 104 104 104 T T T 28 560-579 58 58 58 104 104 104 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 60, Max flush: 110 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 104 104 104 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 54 54 54 97 97 97 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 60 60 60 110 110 110 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 61 61 61 111 111 111 F F F 9 180-199 61 61 61 111 111 111 F F F 10 200-219 58 58 58 107 107 107 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 56 56 56 101 101 101 F F F 15 300-319 58 58 58 104 104 104 F F F 16 320-339 60 60 60 110 110 110 F F F 17 340-359 61 61 61 111 111 111 F F F 18 360-379 60 60 60 109 109 109 F F F 19 380-399 61 61 61 111 111 111 F F F 20 400-419 59 59 59 108 108 108 F F F 21 420-439 61 61 61 111 111 111 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 57 57 57 103 103 103 F F F 25 500-519 61 61 61 111 111 111 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 61 61 61 111 111 111 F F F 28 560-579 58 58 58 104 104 104 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 61, Max flush: 111 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.374249794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.419727552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.420516493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.420553968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.420569146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.420585193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429} {"msg":"Crucible 4e34b717-fcd9-4173-94dd-c58708eae9b7 has session id: bf544b53-8395-4846-bc07-9d21d5b79168","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.421010655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Upstairs opts: Upstairs UUID: 4e34b717-fcd9-4173-94dd-c58708eae9b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.421049639Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Crucible stats registered with UUID: 4e34b717-fcd9-4173-94dd-c58708eae9b7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.421067157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452236551Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429} {"msg":"4e34b717-fcd9-4173-94dd-c58708eae9b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452276166Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452293943Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452336377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452370943Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452406078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452438893Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452473578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452507504Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452543349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452579544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_connection connected from Ok(127.0.0.1:48227)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452614869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_connection connected from Ok(127.0.0.1:60680)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.4526805Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_connection connected from Ok(127.0.0.1:44038)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.452718635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.453386183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.453412809Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.453428527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.453447144Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.45355077Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.453579056Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454118241Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454144628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454159946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454700521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454725698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454739945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454752134Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454765272Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.45477816Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454790318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454803577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[0]R flush_numbers[0..12]: [104, 94, 97, 97, 103, 93, 110, 107, 111, 111, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454817255Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[0]R generation[0..12]: [58, 52, 54, 54, 57, 52, 60, 58, 61, 61, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454835112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.45484965Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[1]R flush_numbers[0..12]: [104, 94, 97, 97, 103, 93, 110, 107, 111, 111, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454863319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[1]R generation[0..12]: [58, 52, 54, 54, 57, 52, 60, 58, 61, 61, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454876587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454890535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[2]R flush_numbers[0..12]: [104, 94, 97, 97, 103, 93, 110, 107, 111, 111, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454904153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[2]R generation[0..12]: [58, 52, 54, 54, 57, 52, 60, 58, 61, 61, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454917441Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454931309Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454944047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454956915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Next flush: 112","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454976253Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.454989181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"":"downstairs","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455002279Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455014198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"0","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455029385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"1","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455042404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"client":"2","":"downstairs","session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455055552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.45506777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"4e34b717-fcd9-4173-94dd-c58708eae9b7 is now active with session: bf544b53-8395-4846-bc07-9d21d5b79168","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455080569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455092457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429,"session_id":"bf544b53-8395-4846-bc07-9d21d5b79168"} {region info: " block size: 4096 bytes msg":"The guest has finished waiting for activation with:62" sub_volume 0 blocks / extent: 20 ,"v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:39.455110584Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 333 len 28672 data: 5 5 5 4 5 5 4 02/30 Write block 391 len 8192 data: 5 5 03/30 Flush 04/30 Write block 354 len 4096 data: 9 05/30 Read block 510 len 16384 06/30 Read block 302 len 40960 07/30 Write block 298 len 16384 data: 7 7 6 8 08/30 Read block 580 len 16384 09/30 Read block 213 len 4096 10/30 Write block 376 len 8192 data: 9 11 11/30 Read block 74 len 16384 12/30 Write block 355 len 16384 data: 8 7 8 6 13/30 Read block 501 len 28672 14/30 Read block 74 len 8192 15/30 Flush 16/30 Write block 74 len 4096 data: 8 17/30 Read block 263 len 12288 18/30 Write block 361 len 28672 data: 5 6 6 5 5 5 6 19/30 Write block 192 len 20480 data: 3 3 3 3 5 20/30 Read block 321 len 28672 21/30 Flush 22/30 Write block 553 len 16384 data: 4 5 4 6 23/30 Read block 457 len 40960 24/30 Write block 203 len 32768 data: 3 3 4 4 4 5 4 5 25/30 Read block 144 len 40960 26/30 Flush 27/30 Write block 327 len 16384 data: 6 4 6 6 28/30 Read block 466 len 20480 29/30 Read block 47 len 32768 30/30 Read block 397 len 28672 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:39.479153273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1429} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 104 104 104 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 62 62 62 114 114 114 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 60 60 60 110 110 110 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 61 61 61 111 111 111 F F F 9 180-199 62 62 62 114 114 114 F F F 10 200-219 62 62 62 115 115 115 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 62 62 62 113 113 113 F F F 16 320-339 62 62 62 112 112 112 T T T 17 340-359 62 62 62 113 113 113 F F F 18 360-379 62 62 62 114 114 114 F F F 19 380-399 62 62 62 112 112 112 F F F 20 400-419 59 59 59 108 108 108 F F F 21 420-439 61 61 61 111 111 111 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 57 57 57 103 103 103 F F F 25 500-519 61 61 61 111 111 111 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 62 62 62 115 115 115 F F F 28 560-579 58 58 58 104 104 104 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 62, Max flush: 115 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 104 104 104 F F F 1 020-039 52 52 52 94 94 94 F F F 2 040-059 54 54 54 97 97 97 F F F 3 060-079 62 62 62 114 114 114 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 60 60 60 110 110 110 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 61 61 61 111 111 111 F F F 9 180-199 62 62 62 114 114 114 F F F 10 200-219 62 62 62 115 115 115 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 62 62 62 113 113 113 F F F 16 320-339 63 63 63 116 116 116 F F F 17 340-359 62 62 62 113 113 113 F F F 18 360-379 62 62 62 114 114 114 F F F 19 380-399 62 62 62 112 112 112 F F F 20 400-419 59 59 59 108 108 108 F F F 21 420-439 61 61 61 111 111 111 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 57 57 57 103 103 103 F F F 25 500-519 61 61 61 111 111 111 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 62 62 62 115 115 115 F F F 28 560-579 58 58 58 104 104 104 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 63, Max flush: 116 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.737983294Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.782550597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.783273907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.783310402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.783336879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.783351617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437} {"msg":"Crucible f6f26098-e260-48ba-95db-201936c01d25 has session id: 9aa04d22-a420-4275-b5b4-c64967ec5115","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.783770409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Upstairs opts: Upstairs UUID: f6f26098-e260-48ba-95db-201936c01d25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.783810973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Crucible stats registered with UUID: f6f26098-e260-48ba-95db-201936c01d25","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.78383557Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815134277Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437} {"msg":"f6f26098-e260-48ba-95db-201936c01d25 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815177631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815196578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815228653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.8152546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815287675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.8153266Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815359275Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.81539552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815449303Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815482888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_connection connected from Ok(127.0.0.1:60078)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815517393Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_connection connected from Ok(127.0.0.1:43358)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815551389Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_connection connected from Ok(127.0.0.1:43402)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.815589234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.81626563Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.816297016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.816325732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.816353618Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.816490539Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.816529164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817079748Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817124751Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817157937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817727248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817761774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.81778673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817805708Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817820275Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817834004Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817846342Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.81785998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[0]R flush_numbers[0..12]: [104, 94, 97, 114, 103, 93, 110, 107, 111, 114, 115, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817873848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[0]R generation[0..12]: [58, 52, 54, 62, 57, 52, 60, 58, 61, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817887676Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817901824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[1]R flush_numbers[0..12]: [104, 94, 97, 114, 103, 93, 110, 107, 111, 114, 115, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817915562Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[1]R generation[0..12]: [58, 52, 54, 62, 57, 52, 60, 58, 61, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.8179291Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817947818Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[2]R flush_numbers[0..12]: [104, 94, 97, 114, 103, 93, 110, 107, 111, 114, 115, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817962166Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[2]R generation[0..12]: [58, 52, 54, 62, 57, 52, 60, 58, 61, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817975544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.817989402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Max found gen is 64","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-09-27T03:52:43.818012289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818037935Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818062472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818082209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"":"downstairs","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818095977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818108206Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"0","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818121784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"1","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818134942Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"client":"2","":"downstairs","session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.8181481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818160548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"f6f26098-e260-48ba-95db-201936c01d25 is now active with session: 9aa04d22-a420-4275-b5b4-c64967ec5115","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818173367Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"region info: 2024-09-27T03:52:43.818188675Z" block size: 4096 bytes ,"hostname":" sub_volume 0 blocks / extent: 20 w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437,"session_id":"9aa04d22-a420-4275-b5b4-c64967ec5115"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:43.818206972Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 101 len 8192 data: 5 6 02/30 Write block 554 len 36864 data: 6 5 7 5 6 7 6 6 4 03/30 Write block 42 len 20480 data: 8 8 6 7 8 04/30 Write block 15 len 12288 data: 7 7 3 05/30 Write block 465 len 28672 data: 4 3 2 3 3 4 4 06/30 Write block 53 len 40960 data: 5 4 4 5 5 4 3 3 4 3 07/30 Write block 113 len 4096 data: 4 08/30 Read block 193 len 24576 09/30 Write block 382 len 12288 data: 5 4 4 10/30 Read block 229 len 28672 11/30 Read block 405 len 32768 12/30 Write block 209 len 4096 data: 5 13/30 Write block 476 len 40960 data: 4 4 5 6 6 6 6 7 7 7 14/30 Read block 542 len 16384 15/30 Read block 30 len 16384 16/30 Write block 349 len 20480 data: 11 9 11 11 10 17/30 Read block 249 len 8192 18/30 Read block 15 len 4096 19/30 Write block 192 len 16384 data: 4 4 4 4 20/30 Write block 187 len 20480 data: 4 4 3 3 3 21/30 Write block 568 len 28672 data: 6 7 6 6 6 6 5 22/30 Read block 410 len 32768 23/30 Write block 5 len 12288 data: 4 5 5 24/30 Read block 92 len 32768 25/30 Read block 494 len 36864 26/30 Flush 27/30 Write block 20 len 20480 data: 3 3 3 2 2 28/30 Write block 414 len 24576 data: 8 6 6 7 7 6 29/30 Write block 52 len 20480 data: 5 6 5 5 6 30/30 Read block 435 len 36864 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 6 Done Sent Done false 29 Acked 1028 Write 5 Done Sent Done false 30 Acked 1029 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 3 1 4 Done 4 1 3 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:43.83893007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1437} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 117 117 117 F F F 1 020-039 52 52 52 94 94 94 T T T 2 040-059 64 64 64 117 117 117 T T T 3 060-079 64 64 64 117 117 117 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 64 64 64 117 117 117 F F F 6 120-139 60 60 60 110 110 110 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 61 61 61 111 111 111 F F F 9 180-199 64 64 64 117 117 117 F F F 10 200-219 64 64 64 117 117 117 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 62 62 62 113 113 113 F F F 16 320-339 63 63 63 116 116 116 F F F 17 340-359 64 64 64 117 117 117 F F F 18 360-379 62 62 62 114 114 114 F F F 19 380-399 64 64 64 117 117 117 F F F 20 400-419 59 59 59 108 108 108 T T T 21 420-439 61 61 61 111 111 111 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 64 64 64 117 117 117 F F F 24 480-499 64 64 64 117 117 117 F F F 25 500-519 61 61 61 111 111 111 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 64 64 64 117 117 117 F F F 28 560-579 64 64 64 117 117 117 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 64, Max flush: 117 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 117 117 117 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 65 65 65 118 118 118 F F F 3 060-079 64 64 64 117 117 117 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 64 64 64 117 117 117 F F F 6 120-139 60 60 60 110 110 110 F F F 7 140-159 58 58 58 107 107 107 F F F 8 160-179 61 61 61 111 111 111 F F F 9 180-199 64 64 64 117 117 117 F F F 10 200-219 64 64 64 117 117 117 F F F 11 220-239 58 58 58 107 107 107 F F F 12 240-259 56 56 56 100 100 100 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 62 62 62 113 113 113 F F F 16 320-339 63 63 63 116 116 116 F F F 17 340-359 64 64 64 117 117 117 F F F 18 360-379 62 62 62 114 114 114 F F F 19 380-399 64 64 64 117 117 117 F F F 20 400-419 65 65 65 118 118 118 F F F 21 420-439 61 61 61 111 111 111 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 64 64 64 117 117 117 F F F 24 480-499 64 64 64 117 117 117 F F F 25 500-519 61 61 61 111 111 111 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 64 64 64 117 117 117 F F F 28 560-579 64 64 64 117 117 117 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 65, Max flush: 118 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.120369826Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:48.163797798Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.164229788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.196150158Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.196844043Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.196880788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.196895146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.196910064Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445} {"msg":"Crucible 9152ebef-f5a9-4578-9829-65b386f31aa4 has session id: 1a38149b-3c2b-4f46-81e8-d7770d8cf814","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.19744198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Upstairs opts: Upstairs UUID: 9152ebef-f5a9-4578-9829-65b386f31aa4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.197485434Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Crucible stats registered with UUID: 9152ebef-f5a9-4578-9829-65b386f31aa4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.197506861Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229615056Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445} {"msg":"9152ebef-f5a9-4578-9829-65b386f31aa4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229676087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229710883Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229754167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229773424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.22980201Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229826517Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229866091Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229899407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229932352Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.229963648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_connection connected from Ok(127.0.0.1:36850)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.230001742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_connection connected from Ok(127.0.0.1:60712)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.230045927Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_connection connected from Ok(127.0.0.1:52419)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.230351474Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.230864214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.230892339Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.230907907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.230921906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.23117833Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.231216395Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.231750331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.231792625Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.231876574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232366296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232390483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.23240505Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232417759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232431807Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232445215Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232457563Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232471012Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[0]R flush_numbers[0..12]: [117, 118, 118, 117, 103, 117, 110, 107, 111, 117, 117, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.23248507Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[0]R generation[0..12]: [64, 65, 65, 64, 57, 64, 60, 58, 61, 64, 64, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232498828Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232512986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[1]R flush_numbers[0..12]: [117, 118, 118, 117, 103, 117, 110, 107, 111, 117, 117, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232526934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[1]R generation[0..12]: [64, 65, 65, 64, 57, 64, 60, 58, 61, 64, 64, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232540542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.2325546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[2]R flush_numbers[0..12]: [117, 118, 118, 117, 103, 117, 110, 107, 111, 117, 117, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232572738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[2]R generation[0..12]: [64, 65, 65, 64, 57, 64, 60, 58, 61, 64, 64, 58]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232587076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232601174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232614192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.23262767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232649327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232662355Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"":"downstairs","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232675553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232687522Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"0","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.2327006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"1","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232713898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"client":"2","":"downstairs","session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232727626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232740244Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"9152ebef-f5a9-4578-9829-65b386f31aa4 is now active with session: 1a38149b-3c2b-4f46-81e8-d7770d8cf814","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232753133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232765261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445,"session_id":"1a38149b-3c2b-4f46-81e8-d7770d8cf814"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:48.232777609Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 49 len 28672 data: 7 5 7 6 7 6 6 02/30 Read block 221 len 8192 03/30 Flush 04/30 Write block 114 len 8192 data: 4 6 05/30 Write block 373 len 16384 data: 7 6 7 10 06/30 Write block 238 len 12288 data: 5 5 5 07/30 Read block 476 len 36864 08/30 Write block 174 len 40960 data: 4 4 5 7 5 6 5 7 7 8 09/30 Read block 408 len 16384 10/30 Read block 249 len 4096 11/30 Write block 558 len 4096 data: 7 12/30 Read block 118 len 4096 13/30 Read block 18 len 20480 14/30 Write block 265 len 28672 data: 6 4 5 6 5 5 6 15/30 Read block 170 len 12288 16/30 Write block 190 len 4096 data: 4 17/30 Read block 26 len 20480 18/30 Flush 19/30 Write block 452 len 36864 data: 5 6 6 8 7 9 7 8 7 20/30 Flush 21/30 Write block 151 len 40960 data: 5 5 6 5 4 4 3 4 4 5 22/30 Flush 23/30 Write block 44 len 36864 data: 7 8 9 7 7 8 6 8 7 24/30 Read block 51 len 12288 25/30 Write block 115 len 28672 data: 7 6 6 6 9 9 8 26/30 Read block 72 len 36864 27/30 Read block 295 len 40960 28/30 Write block 313 len 16384 data: 5 4 4 4 29/30 Write block 509 len 8192 data: 3 3 30/30 Write block 467 len 8192 data: 3 4 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 4 Sent Sent Sent false 29 Acked 1028 Write 2 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:48.257338976Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1445} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 117 117 117 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 66 66 66 119 119 119 T T T 3 060-079 64 64 64 117 117 117 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 66 66 66 120 120 120 T T T 6 120-139 60 60 60 110 110 110 T T T 7 140-159 66 66 66 122 122 122 F F F 8 160-179 66 66 66 122 122 122 F F F 9 180-199 66 66 66 120 120 120 F F F 10 200-219 64 64 64 117 117 117 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 66 66 66 120 120 120 F F F 13 260-279 66 66 66 120 120 120 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 62 62 62 113 113 113 T T T 16 320-339 63 63 63 116 116 116 F F F 17 340-359 64 64 64 117 117 117 F F F 18 360-379 66 66 66 120 120 120 F F F 19 380-399 64 64 64 117 117 117 F F F 20 400-419 65 65 65 118 118 118 F F F 21 420-439 61 61 61 111 111 111 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 66 66 66 121 121 121 T T T 24 480-499 64 64 64 117 117 117 F F F 25 500-519 61 61 61 111 111 111 T T T 26 520-539 61 61 61 111 111 111 F F F 27 540-559 66 66 66 120 120 120 F F F 28 560-579 64 64 64 117 117 117 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 66, Max flush: 122 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 117 117 117 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 67 67 67 123 123 123 F F F 3 060-079 64 64 64 117 117 117 F F F 4 080-099 57 57 57 103 103 103 F F F 5 100-119 67 67 67 123 123 123 F F F 6 120-139 67 67 67 123 123 123 F F F 7 140-159 66 66 66 122 122 122 F F F 8 160-179 66 66 66 122 122 122 F F F 9 180-199 66 66 66 120 120 120 F F F 10 200-219 64 64 64 117 117 117 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 66 66 66 120 120 120 F F F 13 260-279 66 66 66 120 120 120 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 63 63 63 116 116 116 F F F 17 340-359 64 64 64 117 117 117 F F F 18 360-379 66 66 66 120 120 120 F F F 19 380-399 64 64 64 117 117 117 F F F 20 400-419 65 65 65 118 118 118 F F F 21 420-439 61 61 61 111 111 111 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 67 67 67 123 123 123 F F F 24 480-499 64 64 64 117 117 117 F F F 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 66 66 66 120 120 120 F F F 28 560-579 64 64 64 117 117 117 F F F 29 580-599 58 58 58 105 105 105 F F F Max gen: 67, Max flush: 123 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.612727409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:52.657561396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.657949153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.689538839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.690202528Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.690239092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.6902588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.690273768Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453} {"msg":"Crucible 2d0739fe-0c6d-4872-b225-c30e8cf461c4 has session id: 4c3dbafd-2082-4baa-b0e0-3f01d4953bfe","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.69076334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Upstairs opts: Upstairs UUID: 2d0739fe-0c6d-4872-b225-c30e8cf461c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.690800695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Crucible stats registered with UUID: 2d0739fe-0c6d-4872-b225-c30e8cf461c4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.69083293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721774876Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453} {"msg":"2d0739fe-0c6d-4872-b225-c30e8cf461c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721810341Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721828029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721855195Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721872853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.72188976Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721908078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721935724Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.721980438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.722005015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.7220366Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:62156)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.722154934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:44435)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.722520293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:44733)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.722563767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723016345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723057529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723088665Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.72312261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723174483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723203709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723742635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723877906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.723906872Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724346021Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724385756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724414991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724483102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724510788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724525436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724538114Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724551513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[0]R flush_numbers[0..12]: [117, 118, 123, 117, 103, 123, 123, 122, 122, 120, 117, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724565511Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[0]R generation[0..12]: [64, 65, 67, 64, 57, 67, 67, 66, 66, 66, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724579319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724593237Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[1]R flush_numbers[0..12]: [117, 118, 123, 117, 103, 123, 123, 122, 122, 120, 117, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724607085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[1]R generation[0..12]: [64, 65, 67, 64, 57, 67, 67, 66, 66, 66, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724620953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724635011Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[2]R flush_numbers[0..12]: [117, 118, 123, 117, 103, 123, 123, 122, 122, 120, 117, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724648779Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[2]R generation[0..12]: [64, 65, 67, 64, 57, 67, 67, 66, 66, 66, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724665637Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724679715Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724692503Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724705682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.72471868Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:52:52.724731718Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"":"downstairs","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724748985Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724761024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"0","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724774182Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"1","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.72478705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"client":"2","":"downstairs","session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724800428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724812637Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"2d0739fe-0c6d-4872-b225-c30e8cf461c4 is now active with session: 4c3dbafd-2082-4baa-b0e0-3f01d4953bfe","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724825335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724837413Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453,"session_id":"4c3dbafd-2082-4baa-b0e0-3f01d4953bfe"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:52.724849711Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 95 len 28672 data: 6 7 5 7 8 7 6 02/30 Write block 360 len 36864 data: 5 6 7 7 6 6 6 7 9 03/30 Write block 360 len 20480 data: 6 7 8 8 7 04/30 Read block 239 len 28672 05/30 Write block 366 len 8192 data: 7 8 06/30 Flush 07/30 Write block 574 len 40960 data: 6 4 3 4 4 5 4 4 6 6 08/30 Read block 161 len 16384 09/30 Write block 194 len 8192 data: 5 5 10/30 Flush 11/30 Write block 387 len 20480 data: 2 5 5 6 6 12/30 Read block 189 len 32768 13/30 Write block 380 len 24576 data: 7 5 6 5 5 2 14/30 Write block 438 len 20480 data: 2 2 2 2 2 15/30 Read block 482 len 20480 16/30 Read block 458 len 4096 17/30 Flush 18/30 Write block 356 len 28672 data: 8 9 7 6 7 8 9 19/30 Read block 183 len 40960 20/30 Read block 273 len 16384 21/30 Write block 157 len 16384 data: 4 5 5 6 22/30 Read block 569 len 20480 23/30 Read block 576 len 4096 24/30 Read block 59 len 40960 25/30 Read block 2 len 28672 26/30 Read block 534 len 36864 27/30 Read block 244 len 20480 28/30 Write block 14 len 12288 data: 7 8 8 29/30 Read block 37 len 28672 30/30 Read block 293 len 40960 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 7 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:52.749929407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1453} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 117 117 117 T T T 1 020-039 65 65 65 118 118 118 F F F 2 040-059 67 67 67 123 123 123 F F F 3 060-079 64 64 64 117 117 117 F F F 4 080-099 68 68 68 124 124 124 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 67 67 67 123 123 123 F F F 7 140-159 66 66 66 122 122 122 T T T 8 160-179 66 66 66 122 122 122 T T T 9 180-199 68 68 68 125 125 125 F F F 10 200-219 64 64 64 117 117 117 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 66 66 66 120 120 120 F F F 13 260-279 66 66 66 120 120 120 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 63 63 63 116 116 116 F F F 17 340-359 64 64 64 117 117 117 T T T 18 360-379 68 68 68 124 124 124 T T T 19 380-399 68 68 68 126 126 126 F F F 20 400-419 65 65 65 118 118 118 F F F 21 420-439 68 68 68 126 126 126 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 67 67 67 123 123 123 F F F 24 480-499 64 64 64 117 117 117 F F F 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 66 66 66 120 120 120 F F F 28 560-579 68 68 68 125 125 125 F F F 29 580-599 68 68 68 125 125 125 F F F Max gen: 68, Max flush: 126 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 127 127 127 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 67 67 67 123 123 123 F F F 3 060-079 64 64 64 117 117 117 F F F 4 080-099 68 68 68 124 124 124 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 67 67 67 123 123 123 F F F 7 140-159 69 69 69 127 127 127 F F F 8 160-179 69 69 69 127 127 127 F F F 9 180-199 68 68 68 125 125 125 F F F 10 200-219 64 64 64 117 117 117 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 66 66 66 120 120 120 F F F 13 260-279 66 66 66 120 120 120 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 63 63 63 116 116 116 F F F 17 340-359 69 69 69 127 127 127 F F F 18 360-379 69 69 69 127 127 127 F F F 19 380-399 68 68 68 126 126 126 F F F 20 400-419 65 65 65 118 118 118 F F F 21 420-439 68 68 68 126 126 126 F F F 22 440-459 68 68 68 126 126 126 F F F 23 460-479 67 67 67 123 123 123 F F F 24 480-499 64 64 64 117 117 117 F F F 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 66 66 66 120 120 120 F F F 28 560-579 68 68 68 125 125 125 F F F 29 580-599 68 68 68 125 125 125 F F F Max gen: 69, Max flush: 127 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.07903105Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.123606273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.124391775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.124428959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.124449157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.124464065Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461} {"msg":"Crucible ca414e4d-7fe2-4541-a7fd-07b013660e6e has session id: afa025f5-0923-4820-a818-176d25551658","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.1249346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Upstairs opts: Upstairs UUID: ca414e4d-7fe2-4541-a7fd-07b013660e6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.124979183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Crucible stats registered with UUID: ca414e4d-7fe2-4541-a7fd-07b013660e6e","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.12500719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156448087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461} {"msg":"ca414e4d-7fe2-4541-a7fd-07b013660e6e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156485931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156503809Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156531485Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156549462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.15656616Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156582958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156600615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156617793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156635041Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.156671326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_connection connected from Ok(127.0.0.1:48512)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157021337Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_connection connected from Ok(127.0.0.1:51290)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.15707107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_connection connected from Ok(127.0.0.1:44678)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157103576Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157805949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157835265Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157850333Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157864551Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157877839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.157891147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.158466418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.158638344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.158684128Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159051477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159075904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159094741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159311851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159338677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159353045Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159365604Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159379382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[0]R flush_numbers[0..12]: [127, 118, 123, 117, 124, 124, 123, 127, 127, 125, 117, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.15939333Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[0]R generation[0..12]: [69, 65, 67, 64, 68, 68, 67, 69, 69, 68, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159407388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159421586Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[1]R flush_numbers[0..12]: [127, 118, 123, 117, 124, 124, 123, 127, 127, 125, 117, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159435314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[1]R generation[0..12]: [69, 65, 67, 64, 68, 68, 67, 69, 69, 68, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159448652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.15946258Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[2]R flush_numbers[0..12]: [127, 118, 123, 117, 124, 124, 123, 127, 127, 125, 117, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159476048Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[2]R generation[0..12]: [69, 65, 67, 64, 68, 68, 67, 69, 69, 68, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159489496Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159503254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159520362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.15953391Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159546808Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159559477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"":"downstairs","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159572565Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159600681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"0","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159626888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"1","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159653754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"client":"2","":"downstairs","session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.15968047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159706237Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"ca414e4d-7fe2-4541-a7fd-07b013660e6e is now active with session: afa025f5-0923-4820-a818-176d25551658","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159733873Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159759509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461,"session_id":"afa025f5-0923-4820-a818-176d25551658"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:52:57.159785026Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 4096 data: 6 02/30 Read block 118 len 40960 03/30 Flush 04/30 Write block 570 len 24576 data: 7 7 7 7 7 5 05/30 Read block 575 len 24576 06/30 Read block 346 len 36864 07/30 Read block 162 len 28672 08/30 Read block 189 len 4096 09/30 Read block 80 len 24576 10/30 Read block 82 len 12288 11/30 Read block 47 len 16384 12/30 Read block 128 len 28672 13/30 Read block 392 len 8192 14/30 Read block 211 len 28672 15/30 Read block 206 len 12288 16/30 Flush 17/30 Write block 324 len 16384 data: 5 6 7 7 18/30 Read block 31 len 24576 19/30 Read block 266 len 36864 20/30 Read block 491 len 20480 21/30 Read block 68 len 16384 22/30 Flush 23/30 Write block 348 len 12288 data: 11 12 10 24/30 Write block 484 len 12288 data: 8 8 6 25/30 Write block 245 len 40960 data: 6 9 9 8 9 8 7 7 7 7 26/30 Write block 458 len 40960 data: 8 9 8 7 5 5 4 5 4 4 27/30 Read block 237 len 36864 28/30 Write block 414 len 36864 data: 9 7 7 8 8 7 6 5 4 29/30 Write block 264 len 40960 data: 7 7 5 6 7 6 6 7 6 5 30/30 Read block 469 len 12288 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Write 10 Done Sent Sent false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:52:57.183411382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1461} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 127 127 127 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 67 67 67 123 123 123 F F F 3 060-079 64 64 64 117 117 117 F F F 4 080-099 68 68 68 124 124 124 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 67 67 67 123 123 123 F F F 7 140-159 69 69 69 127 127 127 F F F 8 160-179 69 69 69 127 127 127 F F F 9 180-199 68 68 68 125 125 125 F F F 10 200-219 70 70 70 128 128 128 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 66 66 66 120 120 120 T T T 13 260-279 66 66 66 120 120 120 T T T 14 280-299 62 62 62 113 113 113 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 70 70 70 130 130 130 F F F 17 340-359 69 69 69 127 127 127 T T T 18 360-379 69 69 69 127 127 127 F F F 19 380-399 68 68 68 126 126 126 F F F 20 400-419 65 65 65 118 118 118 T T T 21 420-439 68 68 68 126 126 126 T T T 22 440-459 68 68 68 126 126 126 T T T 23 460-479 67 67 67 123 123 123 T T T 24 480-499 64 64 64 117 117 117 T T T 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 66 66 66 120 120 120 F F F 28 560-579 70 70 70 129 129 129 F F F 29 580-599 68 68 68 125 125 125 F F F Max gen: 70, Max flush: 130 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 127 127 127 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 67 67 67 123 123 123 F F F 3 060-079 64 64 64 117 117 117 F F F 4 080-099 68 68 68 124 124 124 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 67 67 67 123 123 123 F F F 7 140-159 69 69 69 127 127 127 F F F 8 160-179 69 69 69 127 127 127 F F F 9 180-199 68 68 68 125 125 125 F F F 10 200-219 70 70 70 128 128 128 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 71 71 71 131 131 131 F F F 13 260-279 71 71 71 131 131 131 F F F 14 280-299 62 62 62 113 113 113 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 70 70 70 130 130 130 F F F 17 340-359 71 71 71 131 131 131 F F F 18 360-379 69 69 69 127 127 127 F F F 19 380-399 68 68 68 126 126 126 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 71 71 71 131 131 131 F F F 22 440-459 71 71 71 131 131 131 F F F 23 460-479 71 71 71 131 131 131 F F F 24 480-499 71 71 71 131 131 131 F F F 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 66 66 66 120 120 120 F F F 28 560-579 70 70 70 129 129 129 F F F 29 580-599 68 68 68 125 125 125 F F F Max gen: 71, Max flush: 131 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.51965237Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:01.562699314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.563050655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.594737458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.595407096Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.595460838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.595487305Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.595511461Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469} {"msg":"Crucible 9f8315a4-3922-474e-b76b-4842eb1df9f8 has session id: fddbde88-9b05-40d4-baa2-becb1e65e996","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.596047307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Upstairs opts: Upstairs UUID: 9f8315a4-3922-474e-b76b-4842eb1df9f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.596098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Crucible stats registered with UUID: 9f8315a4-3922-474e-b76b-4842eb1df9f8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.596128736Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628013472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469} {"msg":"9f8315a4-3922-474e-b76b-4842eb1df9f8 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628059375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628078993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628107939Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628137205Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.62817059Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628202206Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628231142Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628258098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628285074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628312531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_connection connected from Ok(127.0.0.1:39135)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628341047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_connection connected from Ok(127.0.0.1:53163)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628377961Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_connection connected from Ok(127.0.0.1:54774)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.628658553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.629194958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.629228224Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.629302244Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.62932816Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.629354516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.629413048Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.629891472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630021144Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630048771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630575828Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630614713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630643438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.63070396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630736725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630753513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630766312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.63077982Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[0]R flush_numbers[0..12]: [127, 118, 123, 117, 124, 124, 123, 127, 127, 125, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.63084721Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[0]R generation[0..12]: [69, 65, 67, 64, 68, 68, 67, 69, 69, 68, 70, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630866628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630881066Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[1]R flush_numbers[0..12]: [127, 118, 123, 117, 124, 124, 123, 127, 127, 125, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630895024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[1]R generation[0..12]: [69, 65, 67, 64, 68, 68, 67, 69, 69, 68, 70, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630908412Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.63092247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[2]R flush_numbers[0..12]: [127, 118, 123, 117, 124, 124, 123, 127, 127, 125, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630936318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[2]R generation[0..12]: [69, 65, 67, 64, 68, 68, 67, 69, 69, 68, 70, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630949976Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630963974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.630976972Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.63099307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631006438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631019526Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"":"downstairs","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-09-27T03:53:01.631032755Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631051222Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"0","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.63106476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"1","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631077909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"client":"2","":"downstairs","session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631091107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631103475Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"9f8315a4-3922-474e-b76b-4842eb1df9f8 is now active with session: fddbde88-9b05-40d4-baa2-becb1e65e996","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631116573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.631128801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469,"session_id":"fddbde88-9b05-40d4-baa2-becb1e65e996"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:01.63114105Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 275 len 4096 data: 5 02/30 Flush 03/30 Write block 567 len 4096 data: 6 04/30 Write block 389 len 40960 data: 6 7 7 6 5 5 5 5 4 4 05/30 Read block 246 len 8192 06/30 Write block 282 len 4096 data: 5 07/30 Write block 56 len 8192 data: 7 6 08/30 Write block 49 len 36864 data: 9 7 9 8 8 7 7 8 7 09/30 Flush 10/30 Write block 558 len 4096 data: 8 11/30 Flush 12/30 Write block 420 len 28672 data: 7 6 5 3 4 5 5 13/30 Write block 87 len 28672 data: 7 8 8 8 8 6 6 14/30 Read block 78 len 36864 15/30 Read block 532 len 36864 16/30 Read block 172 len 4096 17/30 Write block 67 len 20480 data: 10 10 12 11 11 18/30 Write block 272 len 16384 data: 7 6 5 6 19/30 Flush 20/30 Write block 283 len 16384 data: 5 5 7 8 21/30 Read block 37 len 4096 22/30 Write block 496 len 8192 data: 5 7 23/30 Read block 41 len 4096 24/30 Write block 553 len 16384 data: 5 7 6 8 25/30 Read block 410 len 24576 26/30 Read block 572 len 40960 27/30 Read block 287 len 20480 28/30 Read block 58 len 12288 29/30 Write block 573 len 4096 data: 8 30/30 Read block 169 len 28672 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 1 Sent Done Done false 30 Acked 1029 Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 9 11 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:01.653057522Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1469} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 127 127 127 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 72 72 72 135 135 135 F F F 4 080-099 72 72 72 135 135 135 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 67 67 67 123 123 123 F F F 7 140-159 69 69 69 127 127 127 F F F 8 160-179 69 69 69 127 127 127 F F F 9 180-199 68 68 68 125 125 125 F F F 10 200-219 70 70 70 128 128 128 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 71 71 71 131 131 131 F F F 13 260-279 72 72 72 135 135 135 F F F 14 280-299 72 72 72 133 133 133 T T T 15 300-319 67 67 67 123 123 123 F F F 16 320-339 70 70 70 130 130 130 F F F 17 340-359 71 71 71 131 131 131 F F F 18 360-379 69 69 69 127 127 127 F F F 19 380-399 72 72 72 133 133 133 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 72 72 72 135 135 135 F F F 22 440-459 71 71 71 131 131 131 F F F 23 460-479 71 71 71 131 131 131 F F F 24 480-499 71 71 71 131 131 131 T T T 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 72 72 72 134 134 134 T T T 28 560-579 72 72 72 133 133 133 T T T 29 580-599 68 68 68 125 125 125 F F F Max gen: 72, Max flush: 135 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 127 127 127 F F F 1 020-039 65 65 65 118 118 118 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 72 72 72 135 135 135 F F F 4 080-099 72 72 72 135 135 135 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 67 67 67 123 123 123 F F F 7 140-159 69 69 69 127 127 127 F F F 8 160-179 69 69 69 127 127 127 F F F 9 180-199 68 68 68 125 125 125 F F F 10 200-219 70 70 70 128 128 128 F F F 11 220-239 66 66 66 120 120 120 F F F 12 240-259 71 71 71 131 131 131 F F F 13 260-279 72 72 72 135 135 135 F F F 14 280-299 73 73 73 136 136 136 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 70 70 70 130 130 130 F F F 17 340-359 71 71 71 131 131 131 F F F 18 360-379 69 69 69 127 127 127 F F F 19 380-399 72 72 72 133 133 133 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 72 72 72 135 135 135 F F F 22 440-459 71 71 71 131 131 131 F F F 23 460-479 71 71 71 131 131 131 F F F 24 480-499 73 73 73 136 136 136 F F F 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 68 68 68 125 125 125 F F F Max gen: 73, Max flush: 136 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:05.978169967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.024554909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.025395153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.025430189Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.025444497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.025459325Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477} {"msg":"Crucible bdb6fc24-cfc2-4fee-b3e9-b509c9889ef4 has session id: c2131fb6-f25b-4102-a039-3cab3a080e43","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.025890595Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Upstairs opts: Upstairs UUID: bdb6fc24-cfc2-4fee-b3e9-b509c9889ef4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.025932799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Crucible stats registered with UUID: bdb6fc24-cfc2-4fee-b3e9-b509c9889ef4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.025966205Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057213068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477} {"msg":"bdb6fc24-cfc2-4fee-b3e9-b509c9889ef4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057244674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057262851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057290728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057308535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057325673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057342411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057359008Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057378146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057412181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057445426Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_connection connected from Ok(127.0.0.1:54830)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057484951Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_connection connected from Ok(127.0.0.1:63923)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057519426Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_connection connected from Ok(127.0.0.1:64817)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.057829373Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.058425841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.058452857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.058622574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.058670377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.058704912Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.058734538Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.05908053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059352373Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059462558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059705574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059739419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059756287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059922974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059942911Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.059956549Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060043238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060062225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[0]R flush_numbers[0..12]: [127, 118, 133, 135, 135, 124, 123, 127, 127, 125, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060086961Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[0]R generation[0..12]: [69, 65, 72, 72, 72, 68, 67, 69, 69, 68, 70, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060101519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060115847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[1]R flush_numbers[0..12]: [127, 118, 133, 135, 135, 124, 123, 127, 127, 125, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060129986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[1]R generation[0..12]: [69, 65, 72, 72, 72, 68, 67, 69, 69, 68, 70, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060143624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060157831Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[2]R flush_numbers[0..12]: [127, 118, 133, 135, 135, 124, 123, 127, 127, 125, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.06017168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[2]R generation[0..12]: [69, 65, 72, 72, 72, 68, 67, 69, 69, 68, 70, 66]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060185508Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060210454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060237361Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060262487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060294563Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060325068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"":"downstairs","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060344866Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060357424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"0","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060370772Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"1","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060389479Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"client":"2","":"downstairs","session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060415136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060439423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"bdb6fc24-cfc2-4fee-b3e9-b509c9889ef4 is now active with session: c2131fb6-f25b-4102-a039-3cab3a080e43","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060474878Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477,"session_id":"c2131fb6-f25b-4102-a039-3cab3a080e43"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:06.060497325Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 379 len 40960 data: 8 8 6 7 6 6 3 3 3 6 03/30 Read block 265 len 16384 04/30 Read block 273 len 16384 05/30 Read block 313 len 28672 06/30 Read block 359 len 28672 07/30 Read block 355 len 12288 08/30 Read block 520 len 8192 09/30 Write block 202 len 28672 data: 3 4 4 5 5 5 6 10/30 Read block 443 len 4096 11/30 Write block 184 len 12288 data: 7 6 6 12/30 Write block 146 len 20480 data: 9 9 8 8 5 13/30 Flush 14/30 Write block 178 len 24576 data: 6 7 6 8 8 9 15/30 Write block 213 len 40960 data: 6 6 5 6 6 5 5 5 9 8 16/30 Read block 472 len 4096 17/30 Read block 442 len 4096 18/30 Flush 19/30 Write block 448 len 36864 data: 4 4 4 5 6 7 7 9 8 20/30 Read block 80 len 24576 21/30 Write block 224 len 32768 data: 8 7 8 8 8 8 6 5 22/30 Read block 164 len 28672 23/30 Write block 386 len 20480 data: 4 4 7 7 8 24/30 Flush 25/30 Write block 124 len 32768 data: 6 7 8 8 8 8 10 9 26/30 Read block 511 len 4096 27/30 Read block 222 len 32768 28/30 Write block 286 len 16384 data: 9 9 8 5 29/30 Write block 15 len 36864 data: 9 9 4 2 2 4 4 4 3 30/30 Write block 431 len 8192 data: 3 4 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 4 Done Sent Done false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 2 7 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:06.085814498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1477} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 127 127 127 T T T 1 020-039 65 65 65 118 118 118 T T T 2 040-059 72 72 72 133 133 133 F F F 3 060-079 72 72 72 135 135 135 F F F 4 080-099 72 72 72 135 135 135 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 67 67 67 123 123 123 T T T 7 140-159 74 74 74 138 138 138 F F F 8 160-179 74 74 74 139 139 139 F F F 9 180-199 74 74 74 139 139 139 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 74 74 74 140 140 140 F F F 12 240-259 71 71 71 131 131 131 F F F 13 260-279 72 72 72 135 135 135 F F F 14 280-299 73 73 73 136 136 136 T T T 15 300-319 67 67 67 123 123 123 F F F 16 320-339 70 70 70 130 130 130 F F F 17 340-359 71 71 71 131 131 131 F F F 18 360-379 74 74 74 138 138 138 F F F 19 380-399 74 74 74 140 140 140 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 72 72 72 135 135 135 T T T 22 440-459 74 74 74 140 140 140 F F F 23 460-479 71 71 71 131 131 131 F F F 24 480-499 73 73 73 136 136 136 F F F 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 68 68 68 125 125 125 F F F Max gen: 74, Max flush: 140 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 141 141 141 F F F 1 020-039 75 75 75 141 141 141 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 72 72 72 135 135 135 F F F 4 080-099 72 72 72 135 135 135 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 75 75 75 141 141 141 F F F 7 140-159 74 74 74 138 138 138 F F F 8 160-179 74 74 74 139 139 139 F F F 9 180-199 74 74 74 139 139 139 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 74 74 74 140 140 140 F F F 12 240-259 71 71 71 131 131 131 F F F 13 260-279 72 72 72 135 135 135 F F F 14 280-299 75 75 75 141 141 141 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 70 70 70 130 130 130 F F F 17 340-359 71 71 71 131 131 131 F F F 18 360-379 74 74 74 138 138 138 F F F 19 380-399 74 74 74 140 140 140 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 75 75 75 141 141 141 F F F 22 440-459 74 74 74 140 140 140 F F F 23 460-479 71 71 71 131 131 131 F F F 24 480-499 73 73 73 136 136 136 F F F 25 500-519 67 67 67 123 123 123 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 68 68 68 125 125 125 F F F Max gen: 75, Max flush: 141 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.391681051Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:10.43469405Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.435073937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.467545512Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.46849692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.468542144Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.468560212Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.468578019Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485} {"msg":"Crucible bebeee9f-58e1-42c8-9b60-4c5b3a0e1d77 has session id: 185393b5-b48a-4f73-99e3-e43eb87b03b1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.469082149Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Upstairs opts: Upstairs UUID: bebeee9f-58e1-42c8-9b60-4c5b3a0e1d77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.469130413Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Crucible stats registered with UUID: bebeee9f-58e1-42c8-9b60-4c5b3a0e1d77","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.469167028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500557032Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485} {"msg":"bebeee9f-58e1-42c8-9b60-4c5b3a0e1d77 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500617903Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500652308Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500693553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500726408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500760364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500797348Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500829764Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.50086265Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.500896725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501059332Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44450)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501164458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42721)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501211261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40963)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501250546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501766345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.50179851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501815368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501829326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501843074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.501856562Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.502474837Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.502502793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.502518371Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503035929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503062166Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503076674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503163472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503189628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503209775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503228193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.50324846Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[0]R flush_numbers[0..12]: [141, 141, 133, 135, 135, 124, 141, 138, 139, 139, 139, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503273477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[0]R generation[0..12]: [75, 75, 72, 72, 72, 68, 75, 74, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503294423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503314991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[1]R flush_numbers[0..12]: [141, 141, 133, 135, 135, 124, 141, 138, 139, 139, 139, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503335368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[1]R generation[0..12]: [75, 75, 72, 72, 72, 68, 75, 74, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503354825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503375382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[2]R flush_numbers[0..12]: [141, 141, 133, 135, 135, 124, 141, 138, 139, 139, 139, 140]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503396429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[2]R generation[0..12]: [75, 75, 72, 72, 72, 68, 75, 74, 74, 74, 74, 74]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:53:10.503416867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503442993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.50346323Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503483168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503502415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503521842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"":"downstairs","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.50353555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503550028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"0","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503563306Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"1","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503576285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"client":"2","":"downstairs","session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503589453Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503601681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"bebeee9f-58e1-42c8-9b60-4c5b3a0e1d77 is now active with session: 185393b5-b48a-4f73-99e3-e43eb87b03b1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503614629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503626828Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485,"session_id":"185393b5-b48a-4f73-99e3-e43eb87b03b1"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:10.503639106Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 323 len 32768 data: 5 6 7 8 8 5 7 7 02/30 Write block 271 len 36864 data: 8 8 7 6 7 5 5 5 6 03/30 Read block 242 len 28672 04/30 Write block 30 len 20480 data: 5 5 5 7 6 05/30 Write block 448 len 12288 data: 5 5 5 06/30 Read block 123 len 12288 07/30 Write block 83 len 4096 data: 5 08/30 Flush 09/30 Write block 228 len 16384 data: 9 9 7 6 10/30 Write block 71 len 40960 data: 12 11 11 9 8 8 10 8 6 7 11/30 Read block 37 len 40960 12/30 Write block 344 len 12288 data: 6 5 9 13/30 Read block 577 len 4096 14/30 Write block 221 len 4096 data: 10 15/30 Write block 178 len 16384 data: 7 8 7 9 16/30 Read block 265 len 40960 17/30 Read block 54 len 8192 18/30 Read block 248 len 4096 19/30 Read block 343 len 32768 20/30 Read block 127 len 28672 21/30 Read block 95 len 40960 22/30 Write block 581 len 20480 data: 5 7 7 6 5 23/30 Write block 46 len 16384 data: 10 8 8 10 24/30 Write block 506 len 16384 data: 6 4 3 4 25/30 Read block 98 len 40960 26/30 Read block 230 len 4096 27/30 Write block 181 len 8192 data: 10 9 28/30 Write block 276 len 24576 data: 6 6 6 7 5 6 29/30 Read block 483 len 32768 30/30 Read block 98 len 12288 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 9 Acked 1008 Write 4 Done Done Done false 10 Acked 1009 Write 10 Done Done Done false 11 Acked 1010 Read 10 Done Done Done false 12 Acked 1011 Write 3 Done Done Done false 13 Acked 1012 Read 1 Done Done Done false 14 Acked 1013 Write 1 Done Done Done false 15 Acked 1014 Write 4 Done Done Done false 16 Acked 1015 Read 10 Done Done Done false 17 Acked 1016 Read 2 Done Done Done false 18 Acked 1017 Read 1 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 22 21 21 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:10.52474691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1485} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 141 141 141 F F F 1 020-039 76 76 76 142 142 142 F F F 2 040-059 72 72 72 133 133 133 T T T 3 060-079 72 72 72 135 135 135 T T T 4 080-099 76 76 76 142 142 142 T T T 5 100-119 68 68 68 124 124 124 F F F 6 120-139 75 75 75 141 141 141 F F F 7 140-159 74 74 74 138 138 138 F F F 8 160-179 74 74 74 139 139 139 T T T 9 180-199 74 74 74 139 139 139 T T T 10 200-219 74 74 74 139 139 139 F F F 11 220-239 74 74 74 140 140 140 T T T 12 240-259 71 71 71 131 131 131 F F F 13 260-279 76 76 76 142 142 142 T T T 14 280-299 75 75 75 141 141 141 T T T 15 300-319 67 67 67 123 123 123 F F F 16 320-339 76 76 76 142 142 142 F F F 17 340-359 71 71 71 131 131 131 T T T 18 360-379 74 74 74 138 138 138 F F F 19 380-399 74 74 74 140 140 140 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 75 75 75 141 141 141 F F F 22 440-459 76 76 76 142 142 142 F F F 23 460-479 71 71 71 131 131 131 F F F 24 480-499 73 73 73 136 136 136 F F F 25 500-519 67 67 67 123 123 123 T T T 26 520-539 61 61 61 111 111 111 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 68 68 68 125 125 125 T T T Max gen: 76, Max flush: 142 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 141 141 141 F F F 1 020-039 76 76 76 142 142 142 F F F 2 040-059 77 77 77 143 143 143 F F F 3 060-079 77 77 77 143 143 143 F F F 4 080-099 77 77 77 143 143 143 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 75 75 75 141 141 141 F F F 7 140-159 74 74 74 138 138 138 F F F 8 160-179 77 77 77 143 143 143 F F F 9 180-199 77 77 77 143 143 143 F F F 10 200-219 74 74 74 139 139 139 F F F 11 220-239 77 77 77 143 143 143 F F F 12 240-259 71 71 71 131 131 131 F F F 13 260-279 77 77 77 143 143 143 F F F 14 280-299 77 77 77 143 143 143 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 76 76 76 142 142 142 F F F 17 340-359 77 77 77 143 143 143 F F F 18 360-379 74 74 74 138 138 138 F F F 19 380-399 74 74 74 140 140 140 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 75 75 75 141 141 141 F F F 22 440-459 76 76 76 142 142 142 F F F 23 460-479 71 71 71 131 131 131 F F F 24 480-499 73 73 73 136 136 136 F F F 25 500-519 77 77 77 143 143 143 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 77 77 77 143 143 143 F F F Max gen: 77, Max flush: 143 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.938894786Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.984912439Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.985827383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.985875606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.985893854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.985912201Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493} {"msg":"Crucible fa871e28-b299-4756-9f2c-2fd1a04338fc has session id: 9b29f18b-0ebf-4f5f-bb02-fc85a487d040","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.986307847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Upstairs opts: Upstairs UUID: fa871e28-b299-4756-9f2c-2fd1a04338fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.98635694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Crucible stats registered with UUID: fa871e28-b299-4756-9f2c-2fd1a04338fc","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:14.986405953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.01915067Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493} {"msg":"fa871e28-b299-4756-9f2c-2fd1a04338fc active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019199223Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.01922477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019262185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019290521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019316287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019337134Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019358621Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019379188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019403195Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019429531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_connection connected from Ok(127.0.0.1:56603)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019493302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_connection connected from Ok(127.0.0.1:53348)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.01958173Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_connection connected from Ok(127.0.0.1:49597)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.019737759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.020431133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.020466978Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.020495374Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.02052552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.020556896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.020590411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.02124817Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.021284285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.021304842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.021872694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.02190051Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.021916278Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.021997857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022017564Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022031212Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.02204369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022061438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[0]R flush_numbers[0..12]: [141, 142, 143, 143, 143, 124, 141, 138, 143, 143, 139, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022076156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[0]R generation[0..12]: [75, 76, 77, 77, 77, 68, 75, 74, 77, 77, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022089994Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022103922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[1]R flush_numbers[0..12]: [141, 142, 143, 143, 143, 124, 141, 138, 143, 143, 139, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.0221177Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[1]R generation[0..12]: [75, 76, 77, 77, 77, 68, 75, 74, 77, 77, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022131328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022145276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[2]R flush_numbers[0..12]: [141, 142, 143, 143, 143, 124, 141, 138, 143, 143, 139, 143]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022159224Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[2]R generation[0..12]: [75, 76, 77, 77, 77, 68, 75, 74, 77, 77, 74, 77]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022174042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022201719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022214937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022227905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022240813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"":"downstairs","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022254051Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022272519Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"0","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022308444Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"1","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.0223355Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"client":"2","":"downstairs","session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022352578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022377824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"fa871e28-b299-4756-9f2c-2fd1a04338fc is now active with session: 9b29f18b-0ebf-4f5f-bb02-fc85a487d040","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022399711Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022422418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493,"session_id":"9b29f18b-0ebf-4f5f-bb02-fc85a487d040"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:15.022446135Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 176 len 12288 data: 6 8 8 02/30 Read block 114 len 12288 03/30 Read block 519 len 16384 04/30 Read block 366 len 36864 05/30 Write block 227 len 20480 data: 9 10 10 8 7 06/30 Write block 464 len 40960 data: 5 6 5 5 5 4 5 5 4 4 07/30 Read block 59 len 16384 08/30 Write block 151 len 12288 data: 6 6 7 09/30 Flush 10/30 Write block 514 len 8192 data: 7 9 11/30 Write block 244 len 36864 data: 7 7 10 10 9 10 9 8 8 12/30 Read block 456 len 20480 13/30 Read block 592 len 28672 14/30 Write block 222 len 28672 data: 9 6 9 8 9 10 11 15/30 Write block 2 len 24576 data: 5 5 4 5 6 6 16/30 Write block 465 len 32768 data: 7 6 6 6 5 6 6 5 17/30 Read block 490 len 20480 18/30 Write block 226 len 36864 data: 10 11 12 11 9 8 5 4 6 19/30 Read block 230 len 20480 20/30 Flush 21/30 Write block 480 len 28672 data: 7 7 7 8 9 9 7 22/30 Read block 149 len 4096 23/30 Read block 6 len 40960 24/30 Read block 260 len 32768 25/30 Read block 205 len 40960 26/30 Read block 176 len 12288 27/30 Write block 131 len 36864 data: 10 9 8 8 7 10 9 10 11 28/30 Write block 50 len 36864 data: 8 10 9 9 8 8 9 8 5 29/30 Write block 208 len 16384 data: 7 6 7 5 30/30 Write block 469 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 9 Sent Sent Done false 28 Acked 1027 Write 9 Sent Sent Sent false 29 Acked 1028 Write 4 Sent Sent Sent false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 3 11 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:15.046759546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1493} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 145 145 145 F F F 1 020-039 76 76 76 142 142 142 F F F 2 040-059 77 77 77 143 143 143 T T T 3 060-079 77 77 77 143 143 143 F F F 4 080-099 77 77 77 143 143 143 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 75 75 75 141 141 141 T T T 7 140-159 78 78 78 144 144 144 F F F 8 160-179 78 78 78 144 144 144 F F F 9 180-199 77 77 77 143 143 143 F F F 10 200-219 74 74 74 139 139 139 T T T 11 220-239 78 78 78 145 145 145 F F F 12 240-259 78 78 78 145 145 145 F F F 13 260-279 77 77 77 143 143 143 F F F 14 280-299 77 77 77 143 143 143 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 76 76 76 142 142 142 F F F 17 340-359 77 77 77 143 143 143 F F F 18 360-379 74 74 74 138 138 138 F F F 19 380-399 74 74 74 140 140 140 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 75 75 75 141 141 141 F F F 22 440-459 76 76 76 142 142 142 F F F 23 460-479 78 78 78 145 145 145 T T T 24 480-499 73 73 73 136 136 136 T T T 25 500-519 78 78 78 145 145 145 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 77 77 77 143 143 143 F F F Max gen: 78, Max flush: 145 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 145 145 145 F F F 1 020-039 76 76 76 142 142 142 F F F 2 040-059 79 79 79 146 146 146 F F F 3 060-079 77 77 77 143 143 143 F F F 4 080-099 77 77 77 143 143 143 F F F 5 100-119 68 68 68 124 124 124 F F F 6 120-139 79 79 79 146 146 146 F F F 7 140-159 78 78 78 144 144 144 F F F 8 160-179 78 78 78 144 144 144 F F F 9 180-199 77 77 77 143 143 143 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 78 78 78 145 145 145 F F F 13 260-279 77 77 77 143 143 143 F F F 14 280-299 77 77 77 143 143 143 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 76 76 76 142 142 142 F F F 17 340-359 77 77 77 143 143 143 F F F 18 360-379 74 74 74 138 138 138 F F F 19 380-399 74 74 74 140 140 140 F F F 20 400-419 71 71 71 131 131 131 F F F 21 420-439 75 75 75 141 141 141 F F F 22 440-459 76 76 76 142 142 142 F F F 23 460-479 79 79 79 146 146 146 F F F 24 480-499 79 79 79 146 146 146 F F F 25 500-519 78 78 78 145 145 145 F F F 26 520-539 61 61 61 111 111 111 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 77 77 77 143 143 143 F F F Max gen: 79, Max flush: 146 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.363479961Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.40858104Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.409317179Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.409361683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.409388479Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.409414436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501} {"msg":"Crucible 714ceae0-a2f2-4e45-a4d8-f35bd69a6cac has session id: fad71d16-43ae-48c6-8eff-b4f7241d2889","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.409872672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Upstairs opts: Upstairs UUID: 714ceae0-a2f2-4e45-a4d8-f35bd69a6cac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.409903868Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Crucible stats registered with UUID: 714ceae0-a2f2-4e45-a4d8-f35bd69a6cac","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.409919006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441684488Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501} {"msg":"714ceae0-a2f2-4e45-a4d8-f35bd69a6cac active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441724432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441763147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441807851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441842906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441876631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441909727Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441947092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.441982967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.442018982Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.442055277Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_connection connected from Ok(127.0.0.1:51010)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.442146894Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_connection connected from Ok(127.0.0.1:39313)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.442279586Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_connection connected from Ok(127.0.0.1:39981)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.442325099Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.442945884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.442980719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.443008605Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.44304382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.443076806Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.443155775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.443603283Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.443729515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.443751262Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444291928Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444323723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444340181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444408662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444427759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444441407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444454125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444467723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[0]R flush_numbers[0..12]: [145, 142, 146, 143, 143, 124, 146, 144, 144, 143, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.44448955Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[0]R generation[0..12]: [78, 76, 79, 77, 77, 68, 79, 78, 78, 77, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444508308Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444523576Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[1]R flush_numbers[0..12]: [145, 142, 146, 143, 143, 124, 146, 144, 144, 143, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444537584Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[1]R generation[0..12]: [78, 76, 79, 77, 77, 68, 79, 78, 78, 77, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444551202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.44456503Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[2]R flush_numbers[0..12]: [145, 142, 146, 143, 143, 124, 146, 144, 144, 143, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444578918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[2]R generation[0..12]: [78, 76, 79, 77, 77, 68, 79, 78, 78, 77, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444592366Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444606194Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444619393Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444632691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444645779Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444658527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"":"downstairs","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444671735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444683913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"0","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:53:19.444697212Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"1","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444717329Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"client":"2","":"downstairs","session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444730547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444742856Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"714ceae0-a2f2-4e45-a4d8-f35bd69a6cac is now active with session: fad71d16-43ae-48c6-8eff-b4f7241d2889","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444755804Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.444767892Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501,"session_id":"fad71d16-43ae-48c6-8eff-b4f7241d2889"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:19.44478016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 127 len 32768 data: 9 9 9 11 11 10 9 9 02/30 Write block 243 len 4096 data: 5 03/30 Read block 127 len 12288 04/30 Write block 581 len 24576 data: 6 8 8 7 6 5 05/30 Read block 86 len 40960 06/30 Read block 523 len 16384 07/30 Write block 393 len 24576 data: 6 6 6 6 5 5 08/30 Write block 458 len 40960 data: 9 10 9 8 6 6 6 8 7 7 09/30 Read block 475 len 40960 10/30 Read block 160 len 40960 11/30 Read block 573 len 28672 12/30 Write block 504 len 4096 data: 6 13/30 Read block 343 len 36864 14/30 Write block 157 len 20480 data: 5 6 6 7 6 15/30 Write block 115 len 32768 data: 8 7 7 7 10 10 9 8 16/30 Read block 475 len 4096 17/30 Read block 95 len 20480 18/30 Read block 228 len 24576 19/30 Write block 332 len 24576 data: 5 6 6 6 5 6 20/30 Write block 412 len 4096 data: 8 21/30 Write block 190 len 40960 data: 5 4 5 5 6 6 6 5 5 4 22/30 Write block 585 len 32768 data: 7 6 4 4 3 4 5 5 23/30 Flush 24/30 Write block 424 len 36864 data: 5 6 6 5 3 3 3 4 5 25/30 Write block 334 len 36864 data: 7 7 6 7 6 5 5 6 6 26/30 Read block 457 len 20480 27/30 Read block 470 len 12288 28/30 Read block 262 len 16384 29/30 Read block 201 len 40960 30/30 Write block 513 len 40960 data: 5 8 10 8 7 6 7 7 6 6 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:19.468054545Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1501} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 145 145 145 F F F 1 020-039 76 76 76 142 142 142 F F F 2 040-059 79 79 79 146 146 146 F F F 3 060-079 77 77 77 143 143 143 F F F 4 080-099 77 77 77 143 143 143 F F F 5 100-119 80 80 80 147 147 147 F F F 6 120-139 80 80 80 147 147 147 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 80 80 80 147 147 147 F F F 9 180-199 80 80 80 147 147 147 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 80 80 80 147 147 147 F F F 13 260-279 77 77 77 143 143 143 F F F 14 280-299 77 77 77 143 143 143 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 80 80 80 147 147 147 T T T 17 340-359 77 77 77 143 143 143 T T T 18 360-379 74 74 74 138 138 138 F F F 19 380-399 80 80 80 147 147 147 F F F 20 400-419 80 80 80 147 147 147 F F F 21 420-439 75 75 75 141 141 141 T T T 22 440-459 80 80 80 147 147 147 F F F 23 460-479 80 80 80 147 147 147 F F F 24 480-499 79 79 79 146 146 146 F F F 25 500-519 80 80 80 147 147 147 T T T 26 520-539 61 61 61 111 111 111 T T T 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 80, Max flush: 147 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 145 145 145 F F F 1 020-039 76 76 76 142 142 142 F F F 2 040-059 79 79 79 146 146 146 F F F 3 060-079 77 77 77 143 143 143 F F F 4 080-099 77 77 77 143 143 143 F F F 5 100-119 80 80 80 147 147 147 F F F 6 120-139 80 80 80 147 147 147 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 80 80 80 147 147 147 F F F 9 180-199 80 80 80 147 147 147 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 80 80 80 147 147 147 F F F 13 260-279 77 77 77 143 143 143 F F F 14 280-299 77 77 77 143 143 143 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 81 81 81 148 148 148 F F F 17 340-359 81 81 81 148 148 148 F F F 18 360-379 74 74 74 138 138 138 F F F 19 380-399 80 80 80 147 147 147 F F F 20 400-419 80 80 80 147 147 147 F F F 21 420-439 81 81 81 148 148 148 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 80 80 80 147 147 147 F F F 24 480-499 79 79 79 146 146 146 F F F 25 500-519 81 81 81 148 148 148 F F F 26 520-539 81 81 81 148 148 148 F F F 27 540-559 73 73 73 136 136 136 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 81, Max flush: 148 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.80438493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:23.849133219Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.849624121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.882858551Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.8836624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.883709123Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.883727301Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.883742119Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509} {"msg":"Crucible c8aaaa37-49b1-403d-8a87-e20b005cb57a has session id: 4e00066a-3748-46e7-a084-d55d294c1bb5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.88423623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Upstairs opts: Upstairs UUID: c8aaaa37-49b1-403d-8a87-e20b005cb57a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.884278645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Crucible stats registered with UUID: c8aaaa37-49b1-403d-8a87-e20b005cb57a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.884303251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916415945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509} {"msg":"c8aaaa37-49b1-403d-8a87-e20b005cb57a active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916464959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916492865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916535299Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916571644Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916604389Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916632915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916652003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916676349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916708635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.9167408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:64500)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916774816Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35112)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.916810091Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:63753)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.917047318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.917525272Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.917564397Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.917699898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.917728704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918055959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918084695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.91811758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918159574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.91862875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918811974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918835361Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918850379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918863087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918876725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.918890073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919196411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919218548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[0]R flush_numbers[0..12]: [145, 142, 146, 143, 143, 147, 147, 147, 147, 147, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919237196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[0]R generation[0..12]: [78, 76, 79, 77, 77, 80, 80, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919251483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919270231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[1]R flush_numbers[0..12]: [145, 142, 146, 143, 143, 147, 147, 147, 147, 147, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919285119Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[1]R generation[0..12]: [78, 76, 79, 77, 77, 80, 80, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919298957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919313125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[2]R flush_numbers[0..12]: [145, 142, 146, 143, 143, 147, 147, 147, 147, 147, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919326833Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[2]R generation[0..12]: [78, 76, 79, 77, 77, 80, 80, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919340331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919354069Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919367098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919380236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919393104Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919405852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"":"downstairs","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.91941991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:53:23.919432159Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"0","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919452326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"1","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919467904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"client":"2","":"downstairs","session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919481862Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.91949434Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"c8aaaa37-49b1-403d-8a87-e20b005cb57a is now active with session: 4e00066a-3748-46e7-a084-d55d294c1bb5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919507288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919519107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509,"session_id":"4e00066a-3748-46e7-a084-d55d294c1bb5"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:23.919531115Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 289 len 20480 data: 6 6 7 8 9 02/30 Write block 347 len 36864 data: 10 12 13 11 12 12 11 10 9 03/30 Read block 147 len 20480 04/30 Read block 376 len 36864 05/30 Read block 384 len 20480 06/30 Write block 275 len 20480 data: 8 7 7 7 8 07/30 Read block 94 len 36864 08/30 Flush 09/30 Write block 405 len 20480 data: 9 9 10 12 13 10/30 Write block 7 len 16384 data: 7 7 7 9 11/30 Flush 12/30 Write block 118 len 20480 data: 8 11 11 10 9 13/30 Write block 3 len 8192 data: 6 5 14/30 Write block 243 len 24576 data: 6 8 8 11 11 10 15/30 Read block 484 len 24576 16/30 Read block 539 len 20480 17/30 Read block 265 len 32768 18/30 Read block 96 len 36864 19/30 Read block 524 len 20480 20/30 Write block 346 len 28672 data: 10 11 13 14 12 13 13 21/30 Read block 597 len 4096 22/30 Read block 228 len 4096 23/30 Write block 524 len 40960 data: 7 7 6 5 7 6 8 9 9 9 24/30 Read block 49 len 8192 25/30 Write block 487 len 28672 data: 5 4 5 6 6 4 5 26/30 Read block 199 len 32768 27/30 Write block 549 len 28672 data: 5 3 3 3 6 8 7 28/30 Read block 480 len 24576 29/30 Read block 555 len 24576 30/30 Write block 476 len 20480 data: 5 5 6 7 8 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 12 Acked 1011 Write 5 Done Done Done false 13 Acked 1012 Write 2 Done Done Done false 14 Acked 1013 Write 6 Done Done Done false 15 Acked 1014 Read 6 Done Done Done false 16 Acked 1015 Read 5 Done Done Done false 17 Acked 1016 Read 8 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Write 7 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:23.943968629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1509} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 150 150 150 T T T 1 020-039 76 76 76 142 142 142 F F F 2 040-059 79 79 79 146 146 146 F F F 3 060-079 77 77 77 143 143 143 F F F 4 080-099 77 77 77 143 143 143 F F F 5 100-119 80 80 80 147 147 147 T T T 6 120-139 80 80 80 147 147 147 T T T 7 140-159 80 80 80 147 147 147 F F F 8 160-179 80 80 80 147 147 147 F F F 9 180-199 80 80 80 147 147 147 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 80 80 80 147 147 147 T T T 13 260-279 82 82 82 149 149 149 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 81 81 81 148 148 148 F F F 17 340-359 82 82 82 149 149 149 T T T 18 360-379 74 74 74 138 138 138 F F F 19 380-399 80 80 80 147 147 147 F F F 20 400-419 82 82 82 150 150 150 F F F 21 420-439 81 81 81 148 148 148 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 80 80 80 147 147 147 T T T 24 480-499 79 79 79 146 146 146 T T T 25 500-519 81 81 81 148 148 148 F F F 26 520-539 81 81 81 148 148 148 T T T 27 540-559 73 73 73 136 136 136 T T T 28 560-579 73 73 73 136 136 136 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 82, Max flush: 150 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 151 151 151 F F F 1 020-039 76 76 76 142 142 142 F F F 2 040-059 79 79 79 146 146 146 F F F 3 060-079 77 77 77 143 143 143 F F F 4 080-099 77 77 77 143 143 143 F F F 5 100-119 83 83 83 151 151 151 F F F 6 120-139 83 83 83 151 151 151 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 80 80 80 147 147 147 F F F 9 180-199 80 80 80 147 147 147 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 83 83 83 151 151 151 F F F 13 260-279 82 82 82 149 149 149 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 67 67 67 123 123 123 F F F 16 320-339 81 81 81 148 148 148 F F F 17 340-359 83 83 83 151 151 151 F F F 18 360-379 74 74 74 138 138 138 F F F 19 380-399 80 80 80 147 147 147 F F F 20 400-419 82 82 82 150 150 150 F F F 21 420-439 81 81 81 148 148 148 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 83 83 83 151 151 151 F F F 24 480-499 83 83 83 151 151 151 F F F 25 500-519 81 81 81 148 148 148 F F F 26 520-539 83 83 83 151 151 151 F F F 27 540-559 83 83 83 151 151 151 F F F 28 560-579 73 73 73 136 136 136 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 83, Max flush: 151 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.33253221Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.376856127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.377540243Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.377603074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.3776331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.377665715Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517} {"msg":"Crucible d1550698-7f39-43a9-82f8-0d84ce6d7672 has session id: 26a710a8-7b3f-46ea-acac-8583aa6f8edb","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.378022386Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Upstairs opts: Upstairs UUID: d1550698-7f39-43a9-82f8-0d84ce6d7672, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.37806435Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Crucible stats registered with UUID: d1550698-7f39-43a9-82f8-0d84ce6d7672","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.378085357Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409494759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517} {"msg":"d1550698-7f39-43a9-82f8-0d84ce6d7672 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409549801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409581947Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409637849Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409672384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.40970396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409738175Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409769901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409788138Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409818534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409852619Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:65438)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409880305Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62120)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409904012Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:47524)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.409926029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.410597286Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.410624792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.41063955Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.410653169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.410767583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.410801298Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.411323906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.411461677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.411496022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.411893207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.411932572Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.411956089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.411980025Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412004522Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412029038Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412137214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.4121645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[0]R flush_numbers[0..12]: [151, 142, 146, 143, 143, 151, 151, 147, 147, 147, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412180817Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[0]R generation[0..12]: [83, 76, 79, 77, 77, 83, 83, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412195116Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412228471Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[1]R flush_numbers[0..12]: [151, 142, 146, 143, 143, 151, 151, 147, 147, 147, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412243819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[1]R generation[0..12]: [83, 76, 79, 77, 77, 83, 83, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412257497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412271545Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[2]R flush_numbers[0..12]: [151, 142, 146, 143, 143, 151, 151, 147, 147, 147, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412285053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[2]R generation[0..12]: [83, 76, 79, 77, 77, 83, 83, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412298581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412312539Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412338326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412353054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412366252Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.41237917Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"":"downstairs","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412392338Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:53:28.412404297Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"0","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412420384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"1","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412433673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"client":"2","":"downstairs","session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.41245072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412463698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"d1550698-7f39-43a9-82f8-0d84ce6d7672 is now active with session: 26a710a8-7b3f-46ea-acac-8583aa6f8edb","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412476597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412488665Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517,"session_id":"26a710a8-7b3f-46ea-acac-8583aa6f8edb"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:28.412501113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 65 len 36864 data: 6 6 11 11 13 12 13 12 12 02/30 Write block 467 len 4096 data: 8 03/30 Flush 04/30 Write block 566 len 32768 data: 7 7 7 8 8 8 8 9 05/30 Read block 452 len 36864 06/30 Read block 425 len 28672 07/30 Read block 217 len 32768 08/30 Write block 183 len 20480 data: 10 8 7 7 5 09/30 Read block 538 len 36864 10/30 Write block 471 len 8192 data: 7 6 11/30 Read block 300 len 32768 12/30 Read block 6 len 16384 13/30 Read block 279 len 28672 14/30 Write block 115 len 12288 data: 9 8 8 15/30 Flush 16/30 Flush 17/30 Write block 515 len 12288 data: 11 9 8 18/30 Write block 461 len 24576 data: 9 7 7 7 9 8 19/30 Write block 82 len 20480 data: 6 6 5 7 7 20/30 Write block 45 len 36864 data: 9 11 9 9 11 9 11 10 10 21/30 Read block 51 len 24576 22/30 Write block 49 len 12288 data: 12 10 12 23/30 Read block 68 len 24576 24/30 Write block 507 len 32768 data: 5 4 5 4 3 3 6 9 25/30 Write block 174 len 12288 data: 5 5 7 26/30 Write block 379 len 4096 data: 9 27/30 Write block 126 len 16384 data: 9 10 10 10 28/30 Write block 296 len 36864 data: 8 7 8 8 7 9 8 7 10 29/30 Read block 27 len 16384 30/30 Write block 15 len 32768 data: 10 10 5 3 3 5 5 5 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:28.436037092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1517} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 151 151 151 T T T 1 020-039 76 76 76 142 142 142 T T T 2 040-059 79 79 79 146 146 146 T T T 3 060-079 84 84 84 152 152 152 F F F 4 080-099 77 77 77 143 143 143 T T T 5 100-119 84 84 84 153 153 153 F F F 6 120-139 83 83 83 151 151 151 T T T 7 140-159 80 80 80 147 147 147 F F F 8 160-179 80 80 80 147 147 147 T T T 9 180-199 84 84 84 153 153 153 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 83 83 83 151 151 151 F F F 13 260-279 82 82 82 149 149 149 F F F 14 280-299 82 82 82 149 149 149 T T T 15 300-319 67 67 67 123 123 123 T T T 16 320-339 81 81 81 148 148 148 F F F 17 340-359 83 83 83 151 151 151 F F F 18 360-379 74 74 74 138 138 138 T T T 19 380-399 80 80 80 147 147 147 F F F 20 400-419 82 82 82 150 150 150 F F F 21 420-439 81 81 81 148 148 148 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 84 84 84 153 153 153 T T T 24 480-499 83 83 83 151 151 151 F F F 25 500-519 81 81 81 148 148 148 T T T 26 520-539 83 83 83 151 151 151 F F F 27 540-559 83 83 83 151 151 151 F F F 28 560-579 84 84 84 153 153 153 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 84, Max flush: 153 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 154 154 154 F F F 1 020-039 85 85 85 154 154 154 F F F 2 040-059 85 85 85 154 154 154 F F F 3 060-079 84 84 84 152 152 152 F F F 4 080-099 85 85 85 154 154 154 F F F 5 100-119 84 84 84 153 153 153 F F F 6 120-139 85 85 85 154 154 154 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 85 85 85 154 154 154 F F F 9 180-199 84 84 84 153 153 153 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 83 83 83 151 151 151 F F F 13 260-279 82 82 82 149 149 149 F F F 14 280-299 85 85 85 154 154 154 F F F 15 300-319 85 85 85 154 154 154 F F F 16 320-339 81 81 81 148 148 148 F F F 17 340-359 83 83 83 151 151 151 F F F 18 360-379 85 85 85 154 154 154 F F F 19 380-399 80 80 80 147 147 147 F F F 20 400-419 82 82 82 150 150 150 F F F 21 420-439 81 81 81 148 148 148 F F F 22 440-459 80 80 80 147 147 147 F F F 23 460-479 85 85 85 154 154 154 F F F 24 480-499 83 83 83 151 151 151 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 83 83 83 151 151 151 F F F 27 540-559 83 83 83 151 151 151 F F F 28 560-579 84 84 84 153 153 153 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 85, Max flush: 154 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.84799721Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:32.891638392Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.892041276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.9238646Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.924439341Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.924474616Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.924488174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.924502542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525} {"msg":"Crucible 53ae59ae-533d-458a-aff6-b61bd2a471a2 has session id: 6c744f84-62a1-468c-8481-c8a9467e7db9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.924918754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Upstairs opts: Upstairs UUID: 53ae59ae-533d-458a-aff6-b61bd2a471a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.924957279Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Crucible stats registered with UUID: 53ae59ae-533d-458a-aff6-b61bd2a471a2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.924981886Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.955913813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525} {"msg":"53ae59ae-533d-458a-aff6-b61bd2a471a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.955942729Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.955965296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.955991992Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.95600963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956026777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956043395Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956060103Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956134393Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956254396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956286492Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50070)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956316827Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60849)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956347623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37762)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956377019Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.956972607Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.957001203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.957016141Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.957034158Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.957277515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.95730796Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.957704556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.957848186Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958121238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.95825354Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958285496Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958310732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.95839307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958412808Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958426906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958629978Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958648725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[0]R flush_numbers[0..12]: [154, 154, 154, 152, 154, 153, 154, 147, 154, 153, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958672892Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[0]R generation[0..12]: [85, 85, 85, 84, 85, 84, 85, 80, 85, 84, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958691709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958706457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[1]R flush_numbers[0..12]: [154, 154, 154, 152, 154, 153, 154, 147, 154, 153, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958720255Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[1]R generation[0..12]: [85, 85, 85, 84, 85, 84, 85, 80, 85, 84, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958733954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958747582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[2]R flush_numbers[0..12]: [154, 154, 154, 152, 154, 153, 154, 147, 154, 153, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.9587611Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[2]R generation[0..12]: [85, 85, 85, 84, 85, 84, 85, 80, 85, 84, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958774898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958789286Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958802484Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958815582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.95882855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958841619Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"":"downstairs","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:53:32.958854877Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958871515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"0","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958884753Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"1","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958900311Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"client":"2","":"downstairs","session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958913599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958925797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"53ae59ae-533d-458a-aff6-b61bd2a471a2 is now active with session: 6c744f84-62a1-468c-8481-c8a9467e7db9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958938655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958950753Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525,"session_id":"6c744f84-62a1-468c-8481-c8a9467e7db9"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:32.958963052Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 40960 data: 9 11 9 7 8 5 7 7 6 8 02/30 Read block 386 len 8192 03/30 Read block 81 len 24576 04/30 Flush 05/30 Write block 310 len 4096 data: 6 06/30 Flush 07/30 Flush 08/30 Write block 99 len 8192 data: 9 8 09/30 Write block 161 len 28672 data: 7 6 5 5 5 5 6 10/30 Flush 11/30 Write block 447 len 12288 data: 4 6 6 12/30 Flush 13/30 Write block 540 len 32768 data: 7 8 8 8 9 8 9 7 14/30 Write block 369 len 4096 data: 10 15/30 Write block 319 len 4096 data: 6 16/30 Write block 22 len 12288 data: 6 4 3 17/30 Read block 467 len 16384 18/30 Write block 458 len 20480 data: 10 11 10 10 8 19/30 Read block 32 len 4096 20/30 Write block 567 len 24576 data: 8 8 9 9 9 9 21/30 Write block 123 len 16384 data: 6 7 8 10 22/30 Flush 23/30 Flush 24/30 Write block 161 len 20480 data: 8 7 6 6 6 25/30 Write block 479 len 12288 data: 8 9 8 26/30 Write block 261 len 4096 data: 8 27/30 Read block 441 len 4096 28/30 Write block 27 len 32768 data: 4 4 5 6 6 6 8 7 29/30 Read block 483 len 8192 30/30 Write block 186 len 24576 data: 8 6 5 4 6 5 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:32.984399178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1525} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 154 154 154 F F F 1 020-039 86 86 86 160 160 160 T T T 2 040-059 85 85 85 154 154 154 F F F 3 060-079 86 86 86 155 155 155 F F F 4 080-099 86 86 86 158 158 158 F F F 5 100-119 86 86 86 158 158 158 F F F 6 120-139 86 86 86 160 160 160 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 86 86 86 158 158 158 T T T 9 180-199 84 84 84 153 153 153 T T T 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 83 83 83 151 151 151 F F F 13 260-279 82 82 82 149 149 149 T T T 14 280-299 85 85 85 154 154 154 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 81 81 81 148 148 148 F F F 17 340-359 83 83 83 151 151 151 F F F 18 360-379 86 86 86 160 160 160 F F F 19 380-399 80 80 80 147 147 147 F F F 20 400-419 82 82 82 150 150 150 F F F 21 420-439 81 81 81 148 148 148 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 86 86 86 160 160 160 T T T 24 480-499 83 83 83 151 151 151 T T T 25 500-519 85 85 85 154 154 154 F F F 26 520-539 83 83 83 151 151 151 F F F 27 540-559 86 86 86 160 160 160 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 86, Max flush: 160 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 154 154 154 F F F 1 020-039 87 87 87 161 161 161 F F F 2 040-059 85 85 85 154 154 154 F F F 3 060-079 86 86 86 155 155 155 F F F 4 080-099 86 86 86 158 158 158 F F F 5 100-119 86 86 86 158 158 158 F F F 6 120-139 86 86 86 160 160 160 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 83 83 83 151 151 151 F F F 13 260-279 87 87 87 161 161 161 F F F 14 280-299 85 85 85 154 154 154 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 81 81 81 148 148 148 F F F 17 340-359 83 83 83 151 151 151 F F F 18 360-379 86 86 86 160 160 160 F F F 19 380-399 80 80 80 147 147 147 F F F 20 400-419 82 82 82 150 150 150 F F F 21 420-439 81 81 81 148 148 148 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 87 87 87 161 161 161 F F F 24 480-499 87 87 87 161 161 161 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 83 83 83 151 151 151 F F F 27 540-559 86 86 86 160 160 160 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 87, Max flush: 161 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.341199156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.386081057Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.386869148Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.386904853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.386919811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.386934229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533} {"msg":"Crucible ec3dcf98-3d60-40a2-a94b-5a1daf215c2c has session id: 2f0752dc-a116-485f-91ba-538db7927aa8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.38736Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Upstairs opts: Upstairs UUID: ec3dcf98-3d60-40a2-a94b-5a1daf215c2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.387397725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Crucible stats registered with UUID: ec3dcf98-3d60-40a2-a94b-5a1daf215c2c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.387421112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419116633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533} {"msg":"ec3dcf98-3d60-40a2-a94b-5a1daf215c2c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419177865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.41921024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419257824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419294209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419335003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419370498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419403814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419437469Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419474594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419510439Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41304)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419549853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49369)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419584689Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62765)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.419621434Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.420401516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.420429642Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.42044513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.420458748Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.420472196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.420531408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421109908Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421251589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421286214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.42167234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421698937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421713785Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.42188946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421914617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421928855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421941233Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421954511Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[0]R flush_numbers[0..12]: [154, 161, 154, 155, 158, 158, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421968199Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[0]R generation[0..12]: [85, 87, 85, 86, 86, 86, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421981818Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.421995816Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[1]R flush_numbers[0..12]: [154, 161, 154, 155, 158, 158, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422009704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[1]R generation[0..12]: [85, 87, 85, 86, 86, 86, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422026522Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.42204076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[2]R flush_numbers[0..12]: [154, 161, 154, 155, 158, 158, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422054487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[2]R generation[0..12]: [85, 87, 85, 86, 86, 86, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422067956Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422081694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422094722Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.42210787Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422120948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422134207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"":"downstairs","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422147645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422159633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"0","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.42218375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"1","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422197158Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"client":"2","":"downstairs","session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422210236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422222544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"ec3dcf98-3d60-40a2-a94b-5a1daf215c2c is now active with session: 2f0752dc-a116-485f-91ba-538db7927aa8","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.42225438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422267568Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533,"session_id":"2f0752dc-a116-485f-91ba-538db7927aa8"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:37.422279867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 524 len 32768 data: 8 8 7 6 8 7 9 10 02/30 Write block 322 len 28672 data: 5 6 7 8 9 9 6 03/30 Write block 95 len 4096 data: 7 04/30 Read block 161 len 4096 05/30 Write block 115 len 8192 data: 10 9 06/30 Write block 365 len 20480 data: 7 8 9 10 11 07/30 Read block 166 len 20480 08/30 Read block 295 len 4096 09/30 Read block 86 len 20480 10/30 Write block 377 len 40960 data: 12 7 10 9 7 8 7 7 4 5 11/30 Flush 12/30 Write block 52 len 4096 data: 11 13/30 Read block 505 len 36864 14/30 Flush 15/30 Write block 418 len 12288 data: 9 8 8 16/30 Read block 453 len 40960 17/30 Write block 6 len 24576 data: 7 8 8 8 10 6 18/30 Read block 213 len 24576 19/30 Write block 534 len 36864 data: 9 8 8 5 5 5 8 9 9 20/30 Write block 418 len 8192 data: 10 9 21/30 Read block 4 len 8192 22/30 Write block 421 len 4096 data: 7 23/30 Read block 369 len 4096 24/30 Write block 24 len 28672 data: 4 2 2 5 5 6 7 25/30 Write block 482 len 20480 data: 8 9 10 10 8 26/30 Flush 27/30 Write block 531 len 24576 data: 11 10 10 10 9 9 28/30 Read block 354 len 20480 29/30 Read block 473 len 24576 30/30 Read block 238 len 16384 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:37.445967684Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1533} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 164 164 164 F F F 1 020-039 88 88 88 164 164 164 F F F 2 040-059 88 88 88 163 163 163 F F F 3 060-079 86 86 86 155 155 155 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 86 86 86 160 160 160 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 83 83 83 151 151 151 F F F 13 260-279 87 87 87 161 161 161 F F F 14 280-299 85 85 85 154 154 154 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 88 88 88 162 162 162 F F F 17 340-359 83 83 83 151 151 151 F F F 18 360-379 88 88 88 162 162 162 F F F 19 380-399 88 88 88 162 162 162 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 87 87 87 161 161 161 F F F 24 480-499 88 88 88 164 164 164 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 88 88 88 164 164 164 T T T 27 540-559 88 88 88 164 164 164 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 88, Max flush: 164 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 164 164 164 F F F 1 020-039 88 88 88 164 164 164 F F F 2 040-059 88 88 88 163 163 163 F F F 3 060-079 86 86 86 155 155 155 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 86 86 86 160 160 160 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 83 83 83 151 151 151 F F F 13 260-279 87 87 87 161 161 161 F F F 14 280-299 85 85 85 154 154 154 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 88 88 88 162 162 162 F F F 17 340-359 83 83 83 151 151 151 F F F 18 360-379 88 88 88 162 162 162 F F F 19 380-399 88 88 88 162 162 162 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 87 87 87 161 161 161 F F F 24 480-499 88 88 88 164 164 164 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 89 89 89 165 165 165 F F F 27 540-559 88 88 88 164 164 164 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 89, Max flush: 165 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.71106319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.756361752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.757094401Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.757130146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.757153403Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.757171531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541} {"msg":"Crucible d9d3a1d2-92df-45cb-b293-5078f6432ac1 has session id: 85360402-f9be-4994-9d5c-76c68bde35ed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.757600671Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Upstairs opts: Upstairs UUID: d9d3a1d2-92df-45cb-b293-5078f6432ac1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.757642376Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Crucible stats registered with UUID: d9d3a1d2-92df-45cb-b293-5078f6432ac1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.757668982Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.78953436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541} {"msg":"d9d3a1d2-92df-45cb-b293-5078f6432ac1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789582783Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789600931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789630267Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789647854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789664662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.7896811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789697797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789723794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789763048Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789793754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:38175)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.78982455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:63518)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789853876Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:57428)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.789883322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.790781768Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.790811603Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.790827581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.790841219Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.790913439Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.790947595Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.791472832Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.791514606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.791745864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792019997Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.79206336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792092196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792157387Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792183254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792198072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.79221272Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792226528Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[0]R flush_numbers[0..12]: [164, 164, 163, 155, 162, 162, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792292189Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[0]R generation[0..12]: [88, 88, 88, 86, 88, 88, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792310986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792325274Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[1]R flush_numbers[0..12]: [164, 164, 163, 155, 162, 162, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792338873Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[1]R generation[0..12]: [88, 88, 88, 86, 88, 88, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792352351Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792366059Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[2]R flush_numbers[0..12]: [164, 164, 163, 155, 162, 162, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792380327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[2]R generation[0..12]: [88, 88, 88, 86, 88, 88, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792394725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792413102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.79242736Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792440658Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792453657Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792466535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792485962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.79249798Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"0","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792511229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"1","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792524497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"client":"2","":"downstairs","session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792537845Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792550033Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"d9d3a1d2-92df-45cb-b293-5078f6432ac1 is now active with session: 85360402-f9be-4994-9d5c-76c68bde35ed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792562742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.79257467Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541,"session_id":"85360402-f9be-4994-9d5c-76c68bde35ed"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:41.792586888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 555 len 8192 data: 8 9 02/30 Read block 28 len 16384 03/30 Write block 47 len 20480 data: 10 10 13 11 13 04/30 Write block 385 len 4096 data: 5 05/30 Read block 87 len 4096 06/30 Read block 277 len 20480 07/30 Read block 264 len 36864 08/30 Read block 266 len 8192 09/30 Write block 276 len 40960 data: 8 8 8 9 6 7 6 6 6 8 10/30 Read block 282 len 16384 11/30 Write block 256 len 12288 data: 4 4 5 12/30 Read block 259 len 12288 13/30 Write block 271 len 32768 data: 9 9 8 7 9 9 9 9 14/30 Write block 387 len 16384 data: 5 8 8 9 15/30 Read block 404 len 12288 16/30 Write block 334 len 20480 data: 8 8 7 8 7 17/30 Write block 54 len 8192 data: 9 9 18/30 Read block 404 len 12288 19/30 Write block 245 len 28672 data: 9 12 12 11 11 10 9 20/30 Read block 505 len 36864 21/30 Flush 22/30 Flush 23/30 Write block 349 len 12288 data: 15 13 14 24/30 Write block 354 len 28672 data: 11 10 9 10 8 7 8 25/30 Write block 38 len 32768 data: 7 7 7 8 9 9 8 10 26/30 Read block 67 len 40960 27/30 Read block 216 len 28672 28/30 Read block 203 len 40960 29/30 Read block 457 len 20480 30/30 Write block 16 len 36864 data: 11 6 4 4 6 6 7 5 5 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:41.815265846Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1541} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 164 164 164 T T T 1 020-039 88 88 88 164 164 164 T T T 2 040-059 90 90 90 166 166 166 T T T 3 060-079 86 86 86 155 155 155 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 86 86 86 160 160 160 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 90 90 90 166 166 166 F F F 13 260-279 90 90 90 166 166 166 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 83 83 83 151 151 151 T T T 18 360-379 88 88 88 162 162 162 T T T 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 87 87 87 161 161 161 F F F 24 480-499 88 88 88 164 164 164 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 89 89 89 165 165 165 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 90, Max flush: 166 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 167 167 167 F F F 1 020-039 91 91 91 167 167 167 F F F 2 040-059 91 91 91 167 167 167 F F F 3 060-079 86 86 86 155 155 155 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 86 86 86 160 160 160 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 79 79 79 146 146 146 F F F 11 220-239 78 78 78 145 145 145 F F F 12 240-259 90 90 90 166 166 166 F F F 13 260-279 90 90 90 166 166 166 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 91 91 91 167 167 167 F F F 18 360-379 91 91 91 167 167 167 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 87 87 87 161 161 161 F F F 24 480-499 88 88 88 164 164 164 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 89 89 89 165 165 165 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 80 80 80 147 147 147 F F F Max gen: 91, Max flush: 167 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.133086367Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.180042551Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.180905962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.180944996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.180959984Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.180974592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549} {"msg":"Crucible 3c6d9d1d-c0ef-4eb2-9bd4-bf70953fbc39 has session id: 9cbeaaa1-6125-44a5-a8e0-b72e2290abb4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.181440238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Upstairs opts: Upstairs UUID: 3c6d9d1d-c0ef-4eb2-9bd4-bf70953fbc39, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.181481872Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Crucible stats registered with UUID: 3c6d9d1d-c0ef-4eb2-9bd4-bf70953fbc39","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.181506089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.213428749Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549} {"msg":"3c6d9d1d-c0ef-4eb2-9bd4-bf70953fbc39 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.213882767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.213904093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.21392825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.213942568Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.213955726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.213968725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.213991161Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214011949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214029806Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214049704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:61439)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214206052Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:32821)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214251685Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:39809)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214282611Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.2149459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214975365Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.214991383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.21501572Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.215249448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.215275284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.215844376Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.215873381Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.215889959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216548738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216581174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216596832Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.21660964Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216627137Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216641295Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216653794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216667202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[0]R flush_numbers[0..12]: [167, 167, 167, 155, 162, 162, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.21668095Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[0]R generation[0..12]: [91, 91, 91, 86, 88, 88, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216694638Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216708476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[1]R flush_numbers[0..12]: [167, 167, 167, 155, 162, 162, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216722074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[1]R generation[0..12]: [91, 91, 91, 86, 88, 88, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216735573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216749351Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[2]R flush_numbers[0..12]: [167, 167, 167, 155, 162, 162, 160, 147, 161, 161, 146, 145]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216762819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[2]R generation[0..12]: [91, 91, 91, 86, 88, 88, 86, 80, 87, 87, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216776007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216789765Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216802583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:53:46.216819261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216839568Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216863775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"":"downstairs","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216883612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.21689634Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"0","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216909608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"1","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216929706Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"client":"2","":"downstairs","session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216952592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216965981Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"3c6d9d1d-c0ef-4eb2-9bd4-bf70953fbc39 is now active with session: 9cbeaaa1-6125-44a5-a8e0-b72e2290abb4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216978839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.216990677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549,"session_id":"9cbeaaa1-6125-44a5-a8e0-b72e2290abb4"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:46.217002905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 470 len 20480 data: 7 8 7 5 5 02/30 Write block 492 len 4096 data: 5 03/30 Read block 142 len 24576 04/30 Read block 411 len 28672 05/30 Write block 593 len 20480 data: 4 4 4 3 4 06/30 Read block 156 len 36864 07/30 Read block 41 len 20480 08/30 Read block 128 len 32768 09/30 Write block 487 len 16384 data: 6 5 6 7 10/30 Read block 347 len 4096 11/30 Write block 8 len 36864 data: 9 9 11 7 6 7 8 11 12 12/30 Write block 63 len 16384 data: 4 5 7 7 13/30 Read block 62 len 40960 14/30 Write block 216 len 20480 data: 7 7 6 6 6 15/30 Read block 86 len 28672 16/30 Read block 397 len 32768 17/30 Write block 530 len 20480 data: 10 12 11 11 11 18/30 Write block 304 len 36864 data: 11 12 11 10 8 5 7 6 6 19/30 Read block 57 len 8192 20/30 Write block 44 len 32768 data: 9 11 12 11 11 14 12 14 21/30 Write block 126 len 24576 data: 11 11 11 11 12 12 22/30 Write block 204 len 24576 data: 5 6 6 6 8 7 23/30 Read block 573 len 8192 24/30 Write block 0 len 8192 data: 2 2 25/30 Read block 417 len 20480 26/30 Flush 27/30 Write block 222 len 8192 data: 10 7 28/30 Write block 17 len 24576 data: 7 5 5 7 7 8 29/30 Read block 422 len 8192 30/30 Read block 119 len 8192 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:46.239785209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1549} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 168 168 168 T T T 1 020-039 91 91 91 167 167 167 T T T 2 040-059 92 92 92 168 168 168 F F F 3 060-079 92 92 92 168 168 168 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 92 92 92 168 168 168 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 92 92 92 168 168 168 F F F 11 220-239 92 92 92 168 168 168 T T T 12 240-259 90 90 90 166 166 166 F F F 13 260-279 90 90 90 166 166 166 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 92 92 92 168 168 168 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 91 91 91 167 167 167 F F F 18 360-379 91 91 91 167 167 167 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 92 92 92 168 168 168 F F F 24 480-499 92 92 92 168 168 168 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 92 92 92 168 168 168 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 92 92 92 168 168 168 F F F Max gen: 92, Max flush: 168 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 169 169 169 F F F 1 020-039 93 93 93 169 169 169 F F F 2 040-059 92 92 92 168 168 168 F F F 3 060-079 92 92 92 168 168 168 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 92 92 92 168 168 168 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 92 92 92 168 168 168 F F F 11 220-239 93 93 93 169 169 169 F F F 12 240-259 90 90 90 166 166 166 F F F 13 260-279 90 90 90 166 166 166 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 92 92 92 168 168 168 F F F 16 320-339 90 90 90 166 166 166 F F F 17 340-359 91 91 91 167 167 167 F F F 18 360-379 91 91 91 167 167 167 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 86 86 86 160 160 160 F F F 23 460-479 92 92 92 168 168 168 F F F 24 480-499 92 92 92 168 168 168 F F F 25 500-519 85 85 85 154 154 154 F F F 26 520-539 92 92 92 168 168 168 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 92 92 92 168 168 168 F F F Max gen: 93, Max flush: 169 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.525715516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.570001888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.570764713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.570798838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.570813386Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.570827884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557} {"msg":"Crucible 0e593757-39e0-4fb6-8f87-58ac2339df71 has session id: 54351f73-7a13-499e-96bb-29ba55b48436","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.571281051Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Upstairs opts: Upstairs UUID: 0e593757-39e0-4fb6-8f87-58ac2339df71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.571390996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Crucible stats registered with UUID: 0e593757-39e0-4fb6-8f87-58ac2339df71","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.571423062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.603698364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557} {"msg":"0e593757-39e0-4fb6-8f87-58ac2339df71 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.603749517Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.603768784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.60379862Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.603816637Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.603878849Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.603929062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.603980425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604021719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604051185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604079671Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_connection connected from Ok(127.0.0.1:57397)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604109867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_connection connected from Ok(127.0.0.1:44450)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604142132Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_connection connected from Ok(127.0.0.1:42542)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604332776Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604801231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604829937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604968018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.604990645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.605194107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.605227702Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.605480198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.605566195Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.605929635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606139346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606168332Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.60618404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606197159Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606210937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606224265Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606523933Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606543211Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[0]R flush_numbers[0..12]: [169, 169, 168, 168, 162, 162, 168, 147, 161, 161, 168, 169]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606571897Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[0]R generation[0..12]: [93, 93, 92, 92, 88, 88, 92, 80, 87, 87, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606587185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606601403Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[1]R flush_numbers[0..12]: [169, 169, 168, 168, 162, 162, 168, 147, 161, 161, 168, 169]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606615171Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[1]R generation[0..12]: [93, 93, 92, 92, 88, 88, 92, 80, 87, 87, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606628549Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606642427Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[2]R flush_numbers[0..12]: [169, 169, 168, 168, 162, 162, 168, 147, 161, 161, 168, 169]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606655875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[2]R generation[0..12]: [93, 93, 92, 92, 88, 88, 92, 80, 87, 87, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606669033Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606682771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.60669559Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606708918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606721886Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606734514Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"":"downstairs","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606747612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606759521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"0","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606781708Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"1","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606794946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"client":"2","":"downstairs","session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606808094Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606820312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"0e593757-39e0-4fb6-8f87-58ac2339df71 is now active with session: 54351f73-7a13-499e-96bb-29ba55b48436","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606832911Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606844789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557,"session_id":"54351f73-7a13-499e-96bb-29ba55b48436"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:50.606857027Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 268 len 12288 data: 8 7 7 02/30 Read block 529 len 24576 03/30 Read block 549 len 12288 04/30 Write block 41 len 16384 data: 9 10 10 10 05/30 Write block 512 len 4096 data: 4 06/30 Write block 212 len 16384 data: 5 7 7 6 07/30 Flush 08/30 Write block 360 len 20480 data: 9 9 10 9 8 09/30 Write block 108 len 12288 data: 6 5 6 10/30 Flush 11/30 Write block 256 len 8192 data: 5 5 12/30 Flush 13/30 Write block 189 len 36864 data: 5 7 6 6 6 7 7 7 6 14/30 Write block 584 len 12288 data: 8 8 7 15/30 Write block 512 len 4096 data: 5 16/30 Write block 230 len 16384 data: 10 9 6 5 17/30 Write block 43 len 24576 data: 11 11 12 13 12 12 18/30 Write block 62 len 36864 data: 4 5 6 8 8 12 12 14 13 19/30 Write block 321 len 32768 data: 5 6 7 8 9 10 10 7 20/30 Read block 543 len 28672 21/30 Read block 488 len 20480 22/30 Read block 511 len 32768 23/30 Read block 33 len 16384 24/30 Write block 531 len 32768 data: 13 12 12 12 10 10 6 6 25/30 Flush 26/30 Write block 441 len 36864 data: 3 3 3 3 3 3 5 7 7 27/30 Read block 137 len 24576 28/30 Write block 102 len 8192 data: 7 4 29/30 Read block 146 len 4096 30/30 Read block 433 len 8192 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:50.631746729Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1557} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 169 169 169 F F F 1 020-039 93 93 93 169 169 169 F F F 2 040-059 94 94 94 173 173 173 F F F 3 060-079 94 94 94 173 173 173 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 94 94 94 171 171 171 T T T 6 120-139 92 92 92 168 168 168 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 94 94 94 173 173 173 F F F 10 200-219 94 94 94 170 170 170 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 94 94 94 172 172 172 F F F 13 260-279 94 94 94 170 170 170 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 92 92 92 168 168 168 F F F 16 320-339 94 94 94 173 173 173 F F F 17 340-359 91 91 91 167 167 167 F F F 18 360-379 94 94 94 171 171 171 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 86 86 86 160 160 160 T T T 23 460-479 92 92 92 168 168 168 F F F 24 480-499 92 92 92 168 168 168 F F F 25 500-519 94 94 94 173 173 173 F F F 26 520-539 94 94 94 173 173 173 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 94 94 94 173 173 173 F F F Max gen: 94, Max flush: 173 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 169 169 169 F F F 1 020-039 93 93 93 169 169 169 F F F 2 040-059 94 94 94 173 173 173 F F F 3 060-079 94 94 94 173 173 173 F F F 4 080-099 88 88 88 162 162 162 F F F 5 100-119 95 95 95 174 174 174 F F F 6 120-139 92 92 92 168 168 168 F F F 7 140-159 80 80 80 147 147 147 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 94 94 94 173 173 173 F F F 10 200-219 94 94 94 170 170 170 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 94 94 94 172 172 172 F F F 13 260-279 94 94 94 170 170 170 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 92 92 92 168 168 168 F F F 16 320-339 94 94 94 173 173 173 F F F 17 340-359 91 91 91 167 167 167 F F F 18 360-379 94 94 94 171 171 171 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 95 95 95 174 174 174 F F F 23 460-479 92 92 92 168 168 168 F F F 24 480-499 92 92 92 168 168 168 F F F 25 500-519 94 94 94 173 173 173 F F F 26 520-539 94 94 94 173 173 173 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 94 94 94 173 173 173 F F F Max gen: 95, Max flush: 174 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.896934931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.942457403Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.943190681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.943227866Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.943242394Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.943257112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565} {"msg":"Crucible c4b7d610-eac0-44d6-9377-5515a07321ea has session id: 636a7720-0654-49ee-b108-dbbfa85660b7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.943694372Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Upstairs opts: Upstairs UUID: c4b7d610-eac0-44d6-9377-5515a07321ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.943736856Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Crucible stats registered with UUID: c4b7d610-eac0-44d6-9377-5515a07321ea","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.943757043Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.974825262Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565} {"msg":"c4b7d610-eac0-44d6-9377-5515a07321ea active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.974858047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.974876035Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.974904451Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.974928468Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.974946225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.974963443Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.975000318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.975018045Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.975035203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.97505229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:65090)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.975080107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64981)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.975112182Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55665)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.975150547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.975960195Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.97599395Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.976022686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.976050102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.976076099Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.976102665Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.976591798Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.976695753Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.976729999Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977222431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977249437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977264375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977340764Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977361041Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977374719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977386948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977400396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[0]R flush_numbers[0..12]: [169, 169, 173, 173, 162, 174, 168, 147, 161, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977426582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[0]R generation[0..12]: [93, 93, 94, 94, 88, 95, 92, 80, 87, 94, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.97744085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977455168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[1]R flush_numbers[0..12]: [169, 169, 173, 173, 162, 174, 168, 147, 161, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977472696Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[1]R generation[0..12]: [93, 93, 94, 94, 88, 95, 92, 80, 87, 94, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977486704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977500652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[2]R flush_numbers[0..12]: [169, 169, 173, 173, 162, 174, 168, 147, 161, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.97751452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[2]R generation[0..12]: [93, 93, 94, 94, 88, 95, 92, 80, 87, 94, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977528288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977542047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977554925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977568003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977580781Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977593659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"":"downstairs","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977606787Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977618836Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"0","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977632034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"1","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Transition from WaitQuorum to Active","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977651541Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"client":"2","":"downstairs","session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977666369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977680797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"c4b7d610-eac0-44d6-9377-5515a07321ea is now active with session: 636a7720-0654-49ee-b108-dbbfa85660b7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977693755Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977705754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565,"session_id":"636a7720-0654-49ee-b108-dbbfa85660b7"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:54.977717712Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 152 len 4096 data: 7 03/30 Read block 57 len 40960 04/30 Read block 55 len 36864 05/30 Write block 122 len 16384 data: 10 7 8 9 06/30 Write block 198 len 36864 data: 6 5 4 4 4 5 6 7 7 07/30 Read block 345 len 20480 08/30 Write block 84 len 36864 data: 7 9 8 8 9 9 9 9 7 09/30 Read block 386 len 24576 10/30 Read block 304 len 16384 11/30 Write block 113 len 28672 data: 5 5 11 10 9 9 12 12/30 Read block 534 len 36864 13/30 Read block 490 len 24576 14/30 Flush 15/30 Write block 378 len 28672 data: 8 11 10 8 9 8 8 16/30 Flush 17/30 Write block 465 len 40960 data: 10 9 9 7 7 8 9 8 6 6 18/30 Write block 367 len 32768 data: 10 11 12 7 6 8 8 7 19/30 Read block 90 len 8192 20/30 Write block 243 len 32768 data: 7 9 10 13 13 12 12 11 21/30 Flush 22/30 Write block 341 len 16384 data: 7 7 5 7 23/30 Write block 598 len 4096 data: 4 24/30 Flush 25/30 Write block 383 len 24576 data: 9 9 6 6 6 9 26/30 Write block 499 len 36864 data: 7 8 8 9 8 7 5 7 6 27/30 Read block 175 len 36864 28/30 Write block 317 len 32768 data: 6 5 7 5 6 7 8 9 29/30 Write block 498 len 8192 data: 6 8 30/30 Write block 476 len 32768 data: 6 6 7 9 10 9 9 10 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 8 Done Sent Sent false 29 Acked 1028 Write 2 Sent Sent Sent false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 3 8 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:55.00473913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1565} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 169 169 169 F F F 1 020-039 93 93 93 169 169 169 F F F 2 040-059 94 94 94 173 173 173 F F F 3 060-079 94 94 94 173 173 173 F F F 4 080-099 96 96 96 176 176 176 F F F 5 100-119 96 96 96 176 176 176 F F F 6 120-139 96 96 96 176 176 176 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 96 96 96 176 176 176 F F F 10 200-219 96 96 96 176 176 176 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 96 96 96 178 178 178 F F F 13 260-279 94 94 94 170 170 170 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 92 92 92 168 168 168 T T T 16 320-339 94 94 94 173 173 173 T T T 17 340-359 96 96 96 179 179 179 F F F 18 360-379 96 96 96 178 178 178 F F F 19 380-399 96 96 96 177 177 177 T T T 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 95 95 95 174 174 174 F F F 23 460-479 96 96 96 178 178 178 T F T <--- 24 480-499 92 92 92 168 168 168 T T T 25 500-519 94 94 94 173 173 173 T T T 26 520-539 94 94 94 173 173 173 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 96 96 96 179 179 179 F F F Max gen: 96, Max flush: 179 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 169 169 169 F F F 1 020-039 93 93 93 169 169 169 F F F 2 040-059 94 94 94 173 173 173 F F F 3 060-079 94 94 94 173 173 173 F F F 4 080-099 96 96 96 176 176 176 F F F 5 100-119 96 96 96 176 176 176 F F F 6 120-139 96 96 96 176 176 176 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 96 96 96 176 176 176 F F F 10 200-219 96 96 96 176 176 176 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 96 96 96 178 178 178 F F F 13 260-279 94 94 94 170 170 170 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 97 97 97 180 180 180 F F F 16 320-339 97 97 97 180 180 180 F F F 17 340-359 96 96 96 179 179 179 F F F 18 360-379 96 96 96 178 178 178 F F F 19 380-399 97 97 97 180 180 180 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 95 95 95 174 174 174 F F F 23 460-479 97 97 97 180 180 180 F F F 24 480-499 97 97 97 180 180 180 F F F 25 500-519 97 97 97 180 180 180 F F F 26 520-539 94 94 94 173 173 173 F F F 27 540-559 90 90 90 166 166 166 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 96 96 96 179 179 179 F F F Max gen: 97, Max flush: 180 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.331957494Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.375837353Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.376495812Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.376531947Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.376548515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.376568122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573} {"msg":"Crucible a7de47e1-8170-46ff-81ca-f1aabc5817bb has session id: 3f91c189-4add-490b-83c6-45f892902210","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.377039847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Upstairs opts: Upstairs UUID: a7de47e1-8170-46ff-81ca-f1aabc5817bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.377080621Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Crucible stats registered with UUID: a7de47e1-8170-46ff-81ca-f1aabc5817bb","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.377099588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408109495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573} {"msg":"a7de47e1-8170-46ff-81ca-f1aabc5817bb active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408142631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408160288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408188674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408206831Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408234868Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408273552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408315536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408350202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408384677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408417922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_connection connected from Ok(127.0.0.1:48770)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408453617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_connection connected from Ok(127.0.0.1:53935)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408490362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_connection connected from Ok(127.0.0.1:39591)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.408524118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.409326397Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.409353883Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.409369421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.409383359Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.409396777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.409415804Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410003623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410045677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410080903Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410564216Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.4106035Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410632447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410708386Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410737352Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.41075309Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410766098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410779696Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[0]R flush_numbers[0..12]: [169, 169, 173, 173, 176, 176, 176, 176, 161, 176, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410794114Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[0]R generation[0..12]: [93, 93, 94, 94, 96, 96, 96, 96, 87, 96, 96, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410807992Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.4108223Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[1]R flush_numbers[0..12]: [169, 169, 173, 173, 176, 176, 176, 176, 161, 176, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410836528Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[1]R generation[0..12]: [93, 93, 94, 94, 96, 96, 96, 96, 87, 96, 96, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410852256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410866384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[2]R flush_numbers[0..12]: [169, 169, 173, 173, 176, 176, 176, 176, 161, 176, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410884822Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[2]R generation[0..12]: [93, 93, 94, 94, 96, 96, 96, 96, 87, 96, 96, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410899959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410914117Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410927236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:53:59.410940334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410959901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410972849Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"":"downstairs","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410986248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.410998286Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"0","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.411011674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"1","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.411024842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"client":"2","":"downstairs","session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.411038251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.411050559Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"a7de47e1-8170-46ff-81ca-f1aabc5817bb is now active with session: 3f91c189-4add-490b-83c6-45f892902210","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.411063407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.411075635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573,"session_id":"3f91c189-4add-490b-83c6-45f892902210"} region info: { block size: 4096 bytes "msg":" sub_volume 0 blocks / extent: 20 The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:53:59.411098072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 553 len 12288 data: 7 9 9 02/30 Flush 03/30 Write block 73 len 36864 data: 13 10 9 10 12 10 8 9 6 04/30 Write block 376 len 40960 data: 11 13 9 12 11 9 10 10 10 7 05/30 Read block 381 len 32768 06/30 Read block 181 len 20480 07/30 Write block 132 len 16384 data: 11 10 10 8 08/30 Write block 331 len 28672 data: 5 6 7 9 9 8 9 09/30 Read block 185 len 40960 10/30 Flush 11/30 Write block 217 len 4096 data: 8 12/30 Read block 149 len 32768 13/30 Flush 14/30 Write block 157 len 12288 data: 6 7 7 15/30 Write block 537 len 16384 data: 7 7 6 9 16/30 Write block 375 len 20480 data: 8 12 14 10 13 17/30 Write block 100 len 8192 data: 9 7 18/30 Read block 193 len 36864 19/30 Write block 474 len 16384 data: 7 5 7 7 20/30 Read block 307 len 16384 21/30 Read block 582 len 8192 22/30 Write block 372 len 24576 data: 9 9 8 9 13 15 23/30 Write block 308 len 36864 data: 9 6 8 7 7 6 5 5 5 24/30 Read block 49 len 36864 25/30 Write block 71 len 24576 data: 14 13 14 11 10 11 26/30 Read block 462 len 8192 27/30 Write block 8 len 12288 data: 10 10 12 28/30 Read block 434 len 36864 29/30 Write block 37 len 20480 data: 7 8 8 8 10 30/30 Read block 514 len 40960 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 3 Done Done Done false 15 Acked 1014 Write 4 Done Done Done false 16 Acked 1015 Write 5 Done Done Done false 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 5 Done Sent Done false 30 Acked 1029 Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 17 15 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:53:59.43406511Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1573} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 169 169 169 T T T 1 020-039 93 93 93 169 169 169 T T T 2 040-059 94 94 94 173 173 173 T T T 3 060-079 98 98 98 182 182 182 T T T 4 080-099 98 98 98 182 182 182 F F F 5 100-119 96 96 96 176 176 176 T T T 6 120-139 98 98 98 182 182 182 F F F 7 140-159 96 96 96 176 176 176 T T T 8 160-179 87 87 87 161 161 161 F F F 9 180-199 96 96 96 176 176 176 F F F 10 200-219 98 98 98 183 183 183 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 96 96 96 178 178 178 F F F 13 260-279 94 94 94 170 170 170 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 97 97 97 180 180 180 T T T 16 320-339 98 98 98 182 182 182 F F F 17 340-359 96 96 96 179 179 179 F F F 18 360-379 98 98 98 182 182 182 T T T 19 380-399 98 98 98 182 182 182 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 95 95 95 174 174 174 F F F 23 460-479 97 97 97 180 180 180 T T T 24 480-499 97 97 97 180 180 180 F F F 25 500-519 97 97 97 180 180 180 F F F 26 520-539 94 94 94 173 173 173 T T T 27 540-559 98 98 98 181 181 181 T T T 28 560-579 86 86 86 160 160 160 F F F 29 580-599 96 96 96 179 179 179 F F F Max gen: 98, Max flush: 183 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 F F F 1 020-039 99 99 99 184 184 184 F F F 2 040-059 99 99 99 184 184 184 F F F 3 060-079 99 99 99 184 184 184 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 99 99 99 184 184 184 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 96 96 96 176 176 176 F F F 10 200-219 98 98 98 183 183 183 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 96 96 96 178 178 178 F F F 13 260-279 94 94 94 170 170 170 F F F 14 280-299 90 90 90 166 166 166 F F F 15 300-319 99 99 99 184 184 184 F F F 16 320-339 98 98 98 182 182 182 F F F 17 340-359 96 96 96 179 179 179 F F F 18 360-379 99 99 99 184 184 184 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 F F F 22 440-459 95 95 95 174 174 174 F F F 23 460-479 99 99 99 184 184 184 F F F 24 480-499 97 97 97 180 180 180 F F F 25 500-519 97 97 97 180 180 180 F F F 26 520-539 99 99 99 184 184 184 F F F 27 540-559 99 99 99 184 184 184 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 96 96 96 179 179 179 F F F Max gen: 99, Max flush: 184 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.812514457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:03.85679863Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.857282663Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.890031539Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.890727953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.890764868Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.890779216Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.890793364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581} {"msg":"Crucible 8530332f-4cab-412e-9f48-5ac74e57c190 has session id: f1125a22-211b-40d7-bc8a-8c5390dbaf5c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.891241432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Upstairs opts: Upstairs UUID: 8530332f-4cab-412e-9f48-5ac74e57c190, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.891293405Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Crucible stats registered with UUID: 8530332f-4cab-412e-9f48-5ac74e57c190","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.891321991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.92339559Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581} {"msg":"8530332f-4cab-412e-9f48-5ac74e57c190 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923441754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923467991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923514234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923548879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923575176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923607671Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923639637Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923670543Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.92369007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923708577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58614)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923726145Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43787)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.923926127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49425)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.92397914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.924660066Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.924687192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.9247031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.924717168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.924937617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.924966064Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.925461965Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.9254949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.925511718Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926104766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926132152Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.92614717Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926160049Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926173827Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926186985Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926199113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926212321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[0]R flush_numbers[0..12]: [184, 184, 184, 184, 182, 184, 182, 184, 161, 176, 183, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.92622619Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[0]R generation[0..12]: [99, 99, 99, 99, 98, 99, 98, 99, 87, 96, 98, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926239858Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926254026Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[1]R flush_numbers[0..12]: [184, 184, 184, 184, 182, 184, 182, 184, 161, 176, 183, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926267844Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[1]R generation[0..12]: [99, 99, 99, 99, 98, 99, 98, 99, 87, 96, 98, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926281412Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926299629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[2]R flush_numbers[0..12]: [184, 184, 184, 184, 182, 184, 182, 184, 161, 176, 183, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926314937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[2]R generation[0..12]: [99, 99, 99, 99, 98, 99, 98, 99, 87, 96, 98, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926328695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926342683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926355622Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.92636894Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926381908Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926394966Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"":"downstairs","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-09-27T03:54:03.926414693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926427082Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"0","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.92644031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"1","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926453368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"client":"2","":"downstairs","session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926481304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926494333Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"8530332f-4cab-412e-9f48-5ac74e57c190 is now active with session: f1125a22-211b-40d7-bc8a-8c5390dbaf5c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926507391Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926521709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581,"session_id":"f1125a22-211b-40d7-bc8a-8c5390dbaf5c"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:03.926534447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 26 len 28672 data: 3 6 6 7 8 7 7 02/30 Read block 227 len 20480 03/30 Read block 1 len 16384 04/30 Write block 31 len 4096 data: 8 05/30 Flush 06/30 Write block 268 len 36864 data: 9 8 8 10 10 9 8 10 10 07/30 Read block 200 len 16384 08/30 Read block 226 len 16384 09/30 Write block 377 len 16384 data: 16 11 14 12 10/30 Read block 426 len 32768 11/30 Read block 74 len 4096 12/30 Write block 528 len 12288 data: 9 8 11 13/30 Read block 121 len 4096 14/30 Read block 10 len 28672 15/30 Write block 277 len 20480 data: 10 10 10 7 8 16/30 Write block 512 len 28672 data: 6 7 10 12 10 9 7 17/30 Read block 492 len 40960 18/30 Write block 516 len 24576 data: 11 10 8 8 8 7 19/30 Write block 107 len 28672 data: 6 7 6 7 5 5 6 20/30 Read block 19 len 24576 21/30 Write block 275 len 24576 data: 11 11 11 11 11 8 22/30 Read block 15 len 12288 23/30 Read block 375 len 16384 24/30 Write block 422 len 28672 data: 6 4 6 7 7 6 4 25/30 Read block 537 len 24576 26/30 Read block 580 len 12288 27/30 Read block 108 len 36864 28/30 Read block 515 len 40960 29/30 Read block 409 len 12288 30/30 Read block 207 len 12288 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 6 Acked 1005 Write 9 Done Done Done false 7 Acked 1006 Read 4 Done Done Done false 8 Acked 1007 Read 4 Done Done Done false 9 Acked 1008 Write 4 Done Done Done false 10 Acked 1009 Read 8 Done Done Done false 11 Acked 1010 Read 1 Done Done Done false 12 Acked 1011 Write 3 Done Done Done false 13 Acked 1012 Read 1 Done Done Done false 14 Acked 1013 Read 7 Done Done Done false 15 Acked 1014 Write 5 Done Done Done false 16 Acked 1015 Write 7 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 25 24 24 73 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:03.949059905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1581} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 F F F 1 020-039 100 100 100 185 185 185 F F F 2 040-059 99 99 99 184 184 184 F F F 3 060-079 99 99 99 184 184 184 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 99 99 99 184 184 184 T T T 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 96 96 96 176 176 176 F F F 10 200-219 98 98 98 183 183 183 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 96 96 96 178 178 178 F F F 13 260-279 94 94 94 170 170 170 T T T 14 280-299 90 90 90 166 166 166 T T T 15 300-319 99 99 99 184 184 184 F F F 16 320-339 98 98 98 182 182 182 F F F 17 340-359 96 96 96 179 179 179 F F F 18 360-379 99 99 99 184 184 184 T T T 19 380-399 98 98 98 182 182 182 T T T 20 400-419 88 88 88 164 164 164 F F F 21 420-439 88 88 88 164 164 164 T T T 22 440-459 95 95 95 174 174 174 F F F 23 460-479 99 99 99 184 184 184 F F F 24 480-499 97 97 97 180 180 180 F F F 25 500-519 97 97 97 180 180 180 T T T 26 520-539 99 99 99 184 184 184 T T T 27 540-559 99 99 99 184 184 184 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 96 96 96 179 179 179 F F F Max gen: 100, Max flush: 185 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 F F F 1 020-039 100 100 100 185 185 185 F F F 2 040-059 99 99 99 184 184 184 F F F 3 060-079 99 99 99 184 184 184 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 96 96 96 176 176 176 F F F 10 200-219 98 98 98 183 183 183 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 96 96 96 178 178 178 F F F 13 260-279 101 101 101 186 186 186 F F F 14 280-299 101 101 101 186 186 186 F F F 15 300-319 99 99 99 184 184 184 F F F 16 320-339 98 98 98 182 182 182 F F F 17 340-359 96 96 96 179 179 179 F F F 18 360-379 101 101 101 186 186 186 F F F 19 380-399 101 101 101 186 186 186 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 95 95 95 174 174 174 F F F 23 460-479 99 99 99 184 184 184 F F F 24 480-499 97 97 97 180 180 180 F F F 25 500-519 101 101 101 186 186 186 F F F 26 520-539 101 101 101 186 186 186 F F F 27 540-559 99 99 99 184 184 184 F F F 28 560-579 86 86 86 160 160 160 F F F 29 580-599 96 96 96 179 179 179 F F F Max gen: 101, Max flush: 186 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.329897373Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.373901595Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.374512371Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.374548176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.374562364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.374576982Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589} {"msg":"Crucible a98a69d8-bedc-428e-a6da-21291223380b has session id: 376b9bcb-96cd-4ec0-946f-aad8e3d1e734","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.374988315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Upstairs opts: Upstairs UUID: a98a69d8-bedc-428e-a6da-21291223380b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.375030649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Crucible stats registered with UUID: a98a69d8-bedc-428e-a6da-21291223380b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.375055485Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.405845503Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589} {"msg":"a98a69d8-bedc-428e-a6da-21291223380b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.405882647Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.405911563Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.405958317Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.405995232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406035236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406070281Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406105276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406135792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406168728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406202583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_connection connected from Ok(127.0.0.1:56367)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406239398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_connection connected from Ok(127.0.0.1:59827)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406358952Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_connection connected from Ok(127.0.0.1:35613)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.406406685Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407029299Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407056875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407078802Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.4070931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407152062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407209764Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407707655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407749149Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.407861114Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408363615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408391151Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408406069Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408418467Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408432055Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408445353Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408457532Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.40847066Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[0]R flush_numbers[0..12]: [184, 185, 184, 184, 182, 186, 182, 184, 161, 176, 183, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408484628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[0]R generation[0..12]: [99, 100, 99, 99, 98, 101, 98, 99, 87, 96, 98, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408498626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408512534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[1]R flush_numbers[0..12]: [184, 185, 184, 184, 182, 186, 182, 184, 161, 176, 183, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408526232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[1]R generation[0..12]: [99, 100, 99, 99, 98, 101, 98, 99, 87, 96, 98, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.40854345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408558788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[2]R flush_numbers[0..12]: [184, 185, 184, 184, 182, 186, 182, 184, 161, 176, 183, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408572826Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[2]R generation[0..12]: [99, 100, 99, 99, 98, 101, 98, 99, 87, 96, 98, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408586544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408600342Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.40861327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408626508Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:54:08.408639577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408656044Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"":"downstairs","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408669272Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408681251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"0","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408694459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"1","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408707347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"client":"2","":"downstairs","session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408720345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408733404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"a98a69d8-bedc-428e-a6da-21291223380b is now active with session: 376b9bcb-96cd-4ec0-946f-aad8e3d1e734","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408749171Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.40876156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589,"session_id":"376b9bcb-96cd-4ec0-946f-aad8e3d1e734"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:08.408774168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 20480 data: 7 10 9 10 8 02/30 Write block 524 len 28672 data: 9 9 8 7 10 9 12 03/30 Read block 448 len 20480 04/30 Read block 116 len 12288 05/30 Write block 20 len 16384 data: 8 8 9 6 06/30 Read block 582 len 16384 07/30 Read block 43 len 32768 08/30 Flush 09/30 Write block 564 len 24576 data: 6 7 8 9 9 10 10/30 Read block 107 len 12288 11/30 Flush 12/30 Write block 247 len 4096 data: 14 13/30 Read block 198 len 28672 14/30 Read block 591 len 32768 15/30 Write block 58 len 4096 data: 6 16/30 Write block 57 len 4096 data: 9 17/30 Write block 588 len 32768 data: 5 4 5 6 6 5 5 5 18/30 Read block 225 len 12288 19/30 Write block 434 len 36864 data: 2 2 3 3 3 3 3 4 4 20/30 Write block 57 len 40960 data: 10 7 4 4 5 5 6 7 9 9 21/30 Read block 37 len 4096 22/30 Write block 57 len 4096 data: 11 23/30 Flush 24/30 Write block 190 len 28672 data: 8 7 7 7 8 8 8 25/30 Write block 307 len 4096 data: 11 26/30 Write block 353 len 36864 data: 12 12 11 10 11 9 8 10 10 27/30 Write block 205 len 12288 data: 8 8 7 28/30 Write block 7 len 32768 data: 9 11 11 13 8 7 8 9 29/30 Read block 322 len 16384 30/30 Write block 58 len 40960 data: 8 5 5 6 6 7 8 10 10 13 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:08.433056334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1589} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 T T T 1 020-039 102 102 102 187 187 187 F F F 2 040-059 102 102 102 189 189 189 T T T 3 060-079 102 102 102 189 189 189 T T T 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 96 96 96 176 176 176 T T T 10 200-219 98 98 98 183 183 183 T T T 11 220-239 94 94 94 173 173 173 F F F 12 240-259 102 102 102 189 189 189 F F F 13 260-279 101 101 101 186 186 186 F F F 14 280-299 101 101 101 186 186 186 F F F 15 300-319 99 99 99 184 184 184 T T T 16 320-339 98 98 98 182 182 182 F F F 17 340-359 96 96 96 179 179 179 T T T 18 360-379 101 101 101 186 186 186 T T T 19 380-399 102 102 102 187 187 187 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 102 102 102 189 189 189 F F F 22 440-459 102 102 102 189 189 189 F F F 23 460-479 99 99 99 184 184 184 F F F 24 480-499 97 97 97 180 180 180 F F F 25 500-519 101 101 101 186 186 186 F F F 26 520-539 102 102 102 187 187 187 F F F 27 540-559 99 99 99 184 184 184 F F F 28 560-579 102 102 102 188 188 188 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 102, Max flush: 189 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 F F F 1 020-039 102 102 102 187 187 187 F F F 2 040-059 103 103 103 190 190 190 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 103 103 103 190 190 190 F F F 10 200-219 103 103 103 190 190 190 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 102 102 102 189 189 189 F F F 13 260-279 101 101 101 186 186 186 F F F 14 280-299 101 101 101 186 186 186 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 98 98 98 182 182 182 F F F 17 340-359 103 103 103 190 190 190 F F F 18 360-379 103 103 103 190 190 190 F F F 19 380-399 102 102 102 187 187 187 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 102 102 102 189 189 189 F F F 22 440-459 102 102 102 189 189 189 F F F 23 460-479 99 99 99 184 184 184 F F F 24 480-499 97 97 97 180 180 180 F F F 25 500-519 101 101 101 186 186 186 F F F 26 520-539 102 102 102 187 187 187 F F F 27 540-559 99 99 99 184 184 184 F F F 28 560-579 102 102 102 188 188 188 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 103, Max flush: 190 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.77611672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.82070799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.821454477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.821507399Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.821527537Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.821546024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597} {"msg":"Crucible a01089e5-a6e0-4999-9fb8-adc623776db3 has session id: db180090-3c23-4835-a783-fc25dc0adeb9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.821946699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Upstairs opts: Upstairs UUID: a01089e5-a6e0-4999-9fb8-adc623776db3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.821993422Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Crucible stats registered with UUID: a01089e5-a6e0-4999-9fb8-adc623776db3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.822024438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853171166Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597} {"msg":"a01089e5-a6e0-4999-9fb8-adc623776db3 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853200501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853216949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853244295Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853263243Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.85328103Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853298408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853315186Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853335813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853359719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853390565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64649)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853463935Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64246)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853582999Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57474)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.853627263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.854304279Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.854344334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.854375059Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.854404505Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.854433061Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.854459967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855014231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855103729Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855130525Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855752419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855789264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.85581693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855840627Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855857285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855871123Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855884521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855898139Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 190, 190, 182, 186, 182, 184, 161, 190, 190, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855912037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[0]R generation[0..12]: [103, 102, 103, 103, 98, 101, 98, 99, 87, 103, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855925705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855939623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 190, 190, 182, 186, 182, 184, 161, 190, 190, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855953181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[1]R generation[0..12]: [103, 102, 103, 103, 98, 101, 98, 99, 87, 103, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.85596667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855981168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 190, 190, 182, 186, 182, 184, 161, 190, 190, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.855998475Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[2]R generation[0..12]: [103, 102, 103, 103, 98, 101, 98, 99, 87, 103, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856012743Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856026651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856039549Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856052518Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856072825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856085763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"":"downstairs","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856099181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.85611127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"0","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856124168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"1","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856137066Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"client":"2","":"downstairs","session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856150164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856162332Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"a01089e5-a6e0-4999-9fb8-adc623776db3 is now active with session: db180090-3c23-4835-a783-fc25dc0adeb9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856174831Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856186749Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597,"session_id":"db180090-3c23-4835-a783-fc25dc0adeb9"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:12.856199647Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 248 len 8192 data: 13 13 02/30 Write block 203 len 36864 data: 6 7 9 9 8 9 8 8 6 03/30 Read block 2 len 20480 04/30 Read block 415 len 12288 05/30 Write block 482 len 8192 data: 10 11 06/30 Read block 384 len 16384 07/30 Read block 2 len 12288 08/30 Read block 25 len 12288 09/30 Read block 47 len 8192 10/30 Read block 220 len 16384 11/30 Read block 567 len 40960 12/30 Read block 252 len 36864 13/30 Write block 496 len 12288 data: 6 8 7 14/30 Read block 91 len 8192 15/30 Read block 136 len 4096 16/30 Write block 247 len 24576 data: 15 14 14 12 10 9 17/30 Write block 322 len 16384 data: 8 9 10 10 18/30 Read block 292 len 24576 19/30 Write block 296 len 20480 data: 9 8 9 9 8 20/30 Read block 273 len 28672 21/30 Read block 198 len 8192 22/30 Read block 323 len 36864 23/30 Read block 555 len 8192 24/30 Read block 457 len 28672 25/30 Read block 476 len 32768 26/30 Write block 472 len 20480 data: 9 7 8 6 8 27/30 Write block 181 len 24576 data: 11 10 11 9 8 9 28/30 Read block 171 len 20480 29/30 Write block 297 len 4096 data: 9 30/30 Read block 192 len 24576 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 2 Done Done Done false 2 Acked 1001 Write 9 Done Done Done false 3 Acked 1002 Read 5 Done Done Done false 4 Acked 1003 Read 3 Done Done Done false 5 Acked 1004 Write 2 Done Done Done false 6 Acked 1005 Read 4 Done Done Done false 7 Acked 1006 Read 3 Done Done Done false 8 Acked 1007 Read 3 Done Done Done false 9 Acked 1008 Read 2 Done Done Done false 10 Acked 1009 Read 4 Done Done Done false 11 Acked 1010 Read 10 Done Done Done false 12 Acked 1011 Read 9 Done Done Done false 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Read 2 Done Done Done false 15 Acked 1014 Read 1 Done Done Done false 16 Acked 1015 Write 6 Done Done Done false 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Read 6 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 1 Sent Done Done false 30 Acked 1029 Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 28 30 30 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:12.875315057Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1597} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 F F F 1 020-039 102 102 102 187 187 187 F F F 2 040-059 103 103 103 190 190 190 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 103 103 103 190 190 190 T T T 10 200-219 103 103 103 190 190 190 T T T 11 220-239 94 94 94 173 173 173 F F F 12 240-259 102 102 102 189 189 189 T T T 13 260-279 101 101 101 186 186 186 F F F 14 280-299 101 101 101 186 186 186 T T T 15 300-319 103 103 103 190 190 190 T T T 16 320-339 98 98 98 182 182 182 T T T 17 340-359 103 103 103 190 190 190 F F F 18 360-379 103 103 103 190 190 190 F F F 19 380-399 102 102 102 187 187 187 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 102 102 102 189 189 189 F F F 22 440-459 102 102 102 189 189 189 F F F 23 460-479 99 99 99 184 184 184 T T T 24 480-499 97 97 97 180 180 180 T T T 25 500-519 101 101 101 186 186 186 F F F 26 520-539 102 102 102 187 187 187 F F F 27 540-559 99 99 99 184 184 184 F F F 28 560-579 102 102 102 188 188 188 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 103, Max flush: 190 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 F F F 1 020-039 102 102 102 187 187 187 F F F 2 040-059 103 103 103 190 190 190 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 104 104 104 191 191 191 F F F 10 200-219 104 104 104 191 191 191 F F F 11 220-239 94 94 94 173 173 173 F F F 12 240-259 104 104 104 191 191 191 F F F 13 260-279 101 101 101 186 186 186 F F F 14 280-299 104 104 104 191 191 191 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 104 104 104 191 191 191 F F F 17 340-359 103 103 103 190 190 190 F F F 18 360-379 103 103 103 190 190 190 F F F 19 380-399 102 102 102 187 187 187 F F F 20 400-419 88 88 88 164 164 164 F F F 21 420-439 102 102 102 189 189 189 F F F 22 440-459 102 102 102 189 189 189 F F F 23 460-479 104 104 104 191 191 191 F F F 24 480-499 104 104 104 191 191 191 F F F 25 500-519 101 101 101 186 186 186 F F F 26 520-539 102 102 102 187 187 187 F F F 27 540-559 99 99 99 184 184 184 F F F 28 560-579 102 102 102 188 188 188 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 104, Max flush: 191 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.225151346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:17.267869136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.268234775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.300194311Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.300843391Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.300887985Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.300905782Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.30092349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605} {"msg":"Crucible edcfdc59-376a-4e3e-be38-e3ec0c93345a has session id: 89b54c57-bcbf-4470-a9b1-931323147da7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.301450447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Upstairs opts: Upstairs UUID: edcfdc59-376a-4e3e-be38-e3ec0c93345a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.301493581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Crucible stats registered with UUID: edcfdc59-376a-4e3e-be38-e3ec0c93345a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.301518208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.332899133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605} {"msg":"edcfdc59-376a-4e3e-be38-e3ec0c93345a active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.332947066Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.332965504Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333006198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333046553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333081088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333116273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333183994Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333220918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333256984Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333289649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43991)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333322744Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54078)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33335704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61572)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.333389675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.334039385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.334067112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33408271Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.334096697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33422089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.334265334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.334715402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.334745238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.334894077Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33530697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335331217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335345675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335448291Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335466468Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335479846Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335548347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335565834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 190, 190, 182, 186, 182, 184, 161, 191, 191, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33559714Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[0]R generation[0..12]: [103, 102, 103, 103, 98, 101, 98, 99, 87, 104, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335614848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335632415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 190, 190, 182, 186, 182, 184, 161, 191, 191, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335655312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[1]R generation[0..12]: [103, 102, 103, 103, 98, 101, 98, 99, 87, 104, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33567315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335690797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 190, 190, 182, 186, 182, 184, 161, 191, 191, 173]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335707995Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[2]R generation[0..12]: [103, 102, 103, 103, 98, 101, 98, 99, 87, 104, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335725083Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33574238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Max found gen is 105","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335758398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Generation requested: 106 >= found:105","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335774956Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335790854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335806771Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"":"downstairs","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335831128Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335846176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"0","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335862614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"1","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335878891Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"client":"2","":"downstairs","session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335895519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335913706Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"edcfdc59-376a-4e3e-be38-e3ec0c93345a is now active with session: 89b54c57-bcbf-4470-a9b1-931323147da7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335930194Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.335945472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605,"session_id":"89b54c57-bcbf-4470-a9b1-931323147da7"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:17.33596064Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 429 len 40960 data: 4 4 5 6 3 3 3 4 4 4 02/30 Read block 75 len 24576 03/30 Write block 44 len 8192 data: 12 13 04/30 Write block 330 len 16384 data: 8 6 7 8 05/30 Read block 299 len 28672 06/30 Write block 173 len 8192 data: 3 6 07/30 Write block 517 len 28672 data: 11 9 9 9 8 7 8 08/30 Write block 263 len 8192 data: 7 8 09/30 Read block 201 len 36864 10/30 Flush 11/30 Write block 328 len 28672 data: 8 8 9 7 8 9 10 12/30 Write block 255 len 16384 data: 5 6 6 6 13/30 Write block 402 len 20480 data: 5 7 8 10 10 14/30 Write block 206 len 28672 data: 10 9 10 9 9 7 6 15/30 Read block 42 len 28672 16/30 Read block 230 len 16384 17/30 Write block 236 len 36864 data: 5 5 6 6 6 5 5 8 10 18/30 Read block 30 len 32768 19/30 Flush 20/30 Flush 21/30 Write block 245 len 32768 data: 11 14 16 15 15 13 11 10 22/30 Write block 554 len 28672 data: 10 10 10 6 9 8 7 23/30 Write block 387 len 36864 data: 8 11 10 11 9 7 7 7 7 24/30 Write block 529 len 4096 data: 10 25/30 Write block 234 len 24576 data: 7 5 6 6 7 7 26/30 Write block 56 len 4096 data: 10 27/30 Write block 194 len 12288 data: 9 9 9 28/30 Read block 496 len 40960 29/30 Read block 332 len 4096 30/30 Read block 349 len 36864 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:17.359020615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1605} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 F F F 1 020-039 102 102 102 187 187 187 F F F 2 040-059 106 106 106 192 192 192 T T T 3 060-079 103 103 103 190 190 190 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 104 104 104 191 191 191 T T T 10 200-219 106 106 106 193 193 193 F F F 11 220-239 106 106 106 193 193 193 T T T 12 240-259 106 106 106 193 193 193 T T T 13 260-279 106 106 106 192 192 192 F F F 14 280-299 104 104 104 191 191 191 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 106 106 106 193 193 193 F F F 17 340-359 103 103 103 190 190 190 F F F 18 360-379 103 103 103 190 190 190 F F F 19 380-399 102 102 102 187 187 187 T T T 20 400-419 106 106 106 193 193 193 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 102 102 102 189 189 189 F F F 23 460-479 104 104 104 191 191 191 F F F 24 480-499 104 104 104 191 191 191 F F F 25 500-519 106 106 106 192 192 192 F F F 26 520-539 106 106 106 192 192 192 T T T 27 540-559 99 99 99 184 184 184 T T T 28 560-579 102 102 102 188 188 188 T T T 29 580-599 102 102 102 189 189 189 F F F Max gen: 106, Max flush: 193 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 F F F 1 020-039 102 102 102 187 187 187 F F F 2 040-059 107 107 107 194 194 194 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 98 98 98 182 182 182 F F F 7 140-159 99 99 99 184 184 184 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 107 107 107 194 194 194 F F F 10 200-219 106 106 106 193 193 193 F F F 11 220-239 107 107 107 194 194 194 F F F 12 240-259 107 107 107 194 194 194 F F F 13 260-279 106 106 106 192 192 192 F F F 14 280-299 104 104 104 191 191 191 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 106 106 106 193 193 193 F F F 17 340-359 103 103 103 190 190 190 F F F 18 360-379 103 103 103 190 190 190 F F F 19 380-399 107 107 107 194 194 194 F F F 20 400-419 106 106 106 193 193 193 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 102 102 102 189 189 189 F F F 23 460-479 104 104 104 191 191 191 F F F 24 480-499 104 104 104 191 191 191 F F F 25 500-519 106 106 106 192 192 192 F F F 26 520-539 107 107 107 194 194 194 F F F 27 540-559 107 107 107 194 194 194 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 107, Max flush: 194 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.740940073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.78577012Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.786555281Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.786589876Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.786604314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.786618923Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613} {"msg":"Crucible e1c8b3c7-2516-4fc5-936f-6cd309d89c64 has session id: 6ae7b9ac-94b2-4d44-beaa-5868ecf871e9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.78714491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Upstairs opts: Upstairs UUID: e1c8b3c7-2516-4fc5-936f-6cd309d89c64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.787187774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Crucible stats registered with UUID: e1c8b3c7-2516-4fc5-936f-6cd309d89c64","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.787211211Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.81892516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613} {"msg":"e1c8b3c7-2516-4fc5-936f-6cd309d89c64 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.818963265Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.818982202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819012098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819030545Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819047913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.81907136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819096736Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819125362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819154778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819182904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33689)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819310437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34100)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.81935913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37305)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.819397185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820101517Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820136153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.82015398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820170568Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820264265Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820286442Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820787103Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820819188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.820864252Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821351005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821378001Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821392119Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821466309Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821485506Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821499194Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821511533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821524951Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 194, 190, 182, 186, 182, 184, 192, 194, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821538759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[0]R generation[0..12]: [103, 102, 107, 103, 98, 101, 98, 99, 106, 107, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821552487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821566705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 194, 190, 182, 186, 182, 184, 192, 194, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821580293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[1]R generation[0..12]: [103, 102, 107, 103, 98, 101, 98, 99, 106, 107, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821593871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821611539Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 194, 190, 182, 186, 182, 184, 192, 194, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821625687Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[2]R generation[0..12]: [103, 102, 107, 103, 98, 101, 98, 99, 106, 107, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821639305Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821653323Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821666351Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.82167946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821692448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821705376Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"":"downstairs","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821718464Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-09-27T03:54:21.821730592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"0","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.82175017Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"1","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821763398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"client":"2","":"downstairs","session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821776646Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821788794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"e1c8b3c7-2516-4fc5-936f-6cd309d89c64 is now active with session: 6ae7b9ac-94b2-4d44-beaa-5868ecf871e9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821801603Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821816681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613,"session_id":"6ae7b9ac-94b2-4d44-beaa-5868ecf871e9"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:21.821829329Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 24576 data: 9 12 11 12 10 8 02/30 Read block 253 len 36864 03/30 Read block 184 len 16384 04/30 Read block 483 len 24576 05/30 Write block 457 len 40960 data: 10 11 12 11 11 9 8 8 11 10 06/30 Flush 07/30 Write block 345 len 36864 data: 6 11 12 14 16 14 15 14 13 08/30 Read block 165 len 20480 09/30 Write block 351 len 12288 data: 16 15 14 10/30 Read block 279 len 28672 11/30 Read block 459 len 12288 12/30 Write block 321 len 36864 data: 7 9 10 11 11 11 11 9 9 13/30 Write block 266 len 12288 data: 6 7 10 14/30 Write block 246 len 16384 data: 15 17 16 16 15/30 Write block 153 len 32768 data: 8 6 5 5 7 8 8 8 16/30 Write block 139 len 4096 data: 12 17/30 Read block 462 len 8192 18/30 Read block 370 len 4096 19/30 Flush 20/30 Write block 536 len 28672 data: 11 8 8 7 10 10 10 21/30 Read block 450 len 32768 22/30 Write block 359 len 12288 data: 9 11 11 23/30 Flush 24/30 Write block 144 len 36864 data: 8 10 10 10 9 9 6 7 8 25/30 Read block 63 len 40960 26/30 Read block 76 len 36864 27/30 Read block 405 len 32768 28/30 Write block 72 len 16384 data: 14 15 12 11 29/30 Read block 27 len 8192 30/30 Read block 454 len 32768 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:21.848414766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1613} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 F F F 1 020-039 102 102 102 187 187 187 F F F 2 040-059 107 107 107 194 194 194 F F F 3 060-079 103 103 103 190 190 190 T T T 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 108 108 108 196 196 196 F F F 7 140-159 108 108 108 196 196 196 T T T 8 160-179 108 108 108 196 196 196 F F F 9 180-199 107 107 107 194 194 194 F F F 10 200-219 106 106 106 193 193 193 F F F 11 220-239 107 107 107 194 194 194 F F F 12 240-259 108 108 108 196 196 196 F F F 13 260-279 108 108 108 196 196 196 F F F 14 280-299 104 104 104 191 191 191 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 108 108 108 196 196 196 F F F 17 340-359 108 108 108 197 197 197 F F F 18 360-379 108 108 108 197 197 197 F F F 19 380-399 108 108 108 195 195 195 F F F 20 400-419 106 106 106 193 193 193 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 108 108 108 195 195 195 F F F 23 460-479 108 108 108 195 195 195 F F F 24 480-499 104 104 104 191 191 191 F F F 25 500-519 106 106 106 192 192 192 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 108, Max flush: 197 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 F F F 1 020-039 102 102 102 187 187 187 F F F 2 040-059 107 107 107 194 194 194 F F F 3 060-079 109 109 109 198 198 198 F F F 4 080-099 98 98 98 182 182 182 F F F 5 100-119 101 101 101 186 186 186 F F F 6 120-139 108 108 108 196 196 196 F F F 7 140-159 109 109 109 198 198 198 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 107 107 107 194 194 194 F F F 10 200-219 106 106 106 193 193 193 F F F 11 220-239 107 107 107 194 194 194 F F F 12 240-259 108 108 108 196 196 196 F F F 13 260-279 108 108 108 196 196 196 F F F 14 280-299 104 104 104 191 191 191 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 108 108 108 196 196 196 F F F 17 340-359 108 108 108 197 197 197 F F F 18 360-379 108 108 108 197 197 197 F F F 19 380-399 108 108 108 195 195 195 F F F 20 400-419 106 106 106 193 193 193 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 108 108 108 195 195 195 F F F 23 460-479 108 108 108 195 195 195 F F F 24 480-499 104 104 104 191 191 191 F F F 25 500-519 106 106 106 192 192 192 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 109, Max flush: 198 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.118874481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.16557573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.166475855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.1665149Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.166535307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.166551285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621} {"msg":"Crucible 55b65c19-f9f0-440d-8072-18eb9ed8aa4d has session id: 638970eb-2d19-473e-976c-5e21549f4604","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.1670231Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Upstairs opts: Upstairs UUID: 55b65c19-f9f0-440d-8072-18eb9ed8aa4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.167065914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Crucible stats registered with UUID: 55b65c19-f9f0-440d-8072-18eb9ed8aa4d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.16709068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.19930765Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621} {"msg":"55b65c19-f9f0-440d-8072-18eb9ed8aa4d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199356554Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199375761Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199406037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199424434Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199441882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199458899Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199475677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199492235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.1995279Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199560085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_connection connected from Ok(127.0.0.1:58782)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199712804Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_connection connected from Ok(127.0.0.1:43764)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199763037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_connection connected from Ok(127.0.0.1:49804)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.199799412Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.200577715Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.2006148Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.200643136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.200669302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.200712196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.200730794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201212557Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201241973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201284157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201786448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201819073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201836141Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201895373Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.20191362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201928468Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201940946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201954395Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 194, 198, 182, 186, 196, 198, 196, 194, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.201974502Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[0]R generation[0..12]: [103, 102, 107, 109, 98, 101, 108, 109, 108, 107, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.20198864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202002968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 194, 198, 182, 186, 196, 198, 196, 194, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202016876Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[1]R generation[0..12]: [103, 102, 107, 109, 98, 101, 108, 109, 108, 107, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202030574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202044622Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 194, 198, 182, 186, 196, 198, 196, 194, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.20205833Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[2]R generation[0..12]: [103, 102, 107, 109, 98, 101, 108, 109, 108, 107, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202074638Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202089106Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202102454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202115592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.20212836Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202141319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"":"downstairs","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-09-27T03:54:26.202154487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202174574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"0","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202187882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"1","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202200901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"client":"2","":"downstairs","session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202213929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202226527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"55b65c19-f9f0-440d-8072-18eb9ed8aa4d is now active with session: 638970eb-2d19-473e-976c-5e21549f4604","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202239455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202251443Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621,"session_id":"638970eb-2d19-473e-976c-5e21549f4604"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:26.202263592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 283 len 16384 data: 7 7 9 10 02/30 Read block 132 len 8192 03/30 Read block 355 len 40960 04/30 Read block 246 len 32768 05/30 Write block 222 len 36864 data: 11 8 10 9 11 12 13 12 11 06/30 Read block 177 len 32768 07/30 Read block 41 len 12288 08/30 Read block 272 len 40960 09/30 Read block 21 len 12288 10/30 Read block 6 len 12288 11/30 Read block 128 len 8192 12/30 Read block 552 len 16384 13/30 Read block 446 len 12288 14/30 Write block 349 len 20480 data: 17 15 17 16 15 15/30 Flush 16/30 Write block 226 len 20480 data: 12 13 14 13 12 17/30 Flush 18/30 Write block 88 len 4096 data: 10 19/30 Write block 368 len 20480 data: 12 13 8 7 10 20/30 Read block 145 len 4096 21/30 Write block 40 len 28672 data: 9 11 11 12 13 14 14 22/30 Write block 459 len 16384 data: 13 12 12 10 23/30 Write block 113 len 20480 data: 7 6 12 11 10 24/30 Read block 585 len 40960 25/30 Write block 76 len 16384 data: 12 13 11 9 26/30 Write block 22 len 8192 data: 10 7 27/30 Write block 246 len 16384 data: 16 18 17 17 28/30 Write block 152 len 28672 data: 9 9 7 6 6 8 9 29/30 Write block 363 len 36864 data: 10 9 8 9 11 13 14 9 8 30/30 Write block 7 len 8192 data: 10 12 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 7 Sent Done Sent false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 3 8 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:26.224092067Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1621} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 190 190 190 T T T 1 020-039 102 102 102 187 187 187 T T T 2 040-059 107 107 107 194 194 194 T T T 3 060-079 109 109 109 198 198 198 T T T 4 080-099 98 98 98 182 182 182 T T T 5 100-119 101 101 101 186 186 186 T T T 6 120-139 108 108 108 196 196 196 F F F 7 140-159 109 109 109 198 198 198 T T T 8 160-179 108 108 108 196 196 196 F F F 9 180-199 107 107 107 194 194 194 F F F 10 200-219 106 106 106 193 193 193 F F F 11 220-239 110 110 110 200 200 200 F F F 12 240-259 108 108 108 196 196 196 T T T 13 260-279 108 108 108 196 196 196 F F F 14 280-299 110 110 110 199 199 199 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 108 108 108 196 196 196 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 108 108 108 197 197 197 T T T 19 380-399 108 108 108 195 195 195 F F F 20 400-419 106 106 106 193 193 193 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 108 108 108 195 195 195 T T T 23 460-479 108 108 108 195 195 195 T T T 24 480-499 104 104 104 191 191 191 F F F 25 500-519 106 106 106 192 192 192 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 110, Max flush: 200 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 201 201 201 F F F 1 020-039 111 111 111 201 201 201 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 111 111 111 201 201 201 F F F 4 080-099 111 111 111 201 201 201 F F F 5 100-119 111 111 111 201 201 201 F F F 6 120-139 108 108 108 196 196 196 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 107 107 107 194 194 194 F F F 10 200-219 106 106 106 193 193 193 F F F 11 220-239 110 110 110 200 200 200 F F F 12 240-259 111 111 111 201 201 201 F F F 13 260-279 108 108 108 196 196 196 F F F 14 280-299 110 110 110 199 199 199 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 108 108 108 196 196 196 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 111 111 111 201 201 201 F F F 19 380-399 108 108 108 195 195 195 F F F 20 400-419 106 106 106 193 193 193 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 111 111 111 201 201 201 F F F 24 480-499 104 104 104 191 191 191 F F F 25 500-519 106 106 106 192 192 192 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 111, Max flush: 201 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.616275117Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.661070439Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.66185882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.661897485Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.661911713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.66193217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629} {"msg":"Crucible 5c8ad893-45f4-422c-b920-b2021fd3d4f1 has session id: 8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.662384208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Upstairs opts: Upstairs UUID: 5c8ad893-45f4-422c-b920-b2021fd3d4f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.662424672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Crucible stats registered with UUID: 5c8ad893-45f4-422c-b920-b2021fd3d4f1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.662448509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694576011Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629} {"msg":"5c8ad893-45f4-422c-b920-b2021fd3d4f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694608197Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694623945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694645462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694734569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694768015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694797361Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694822227Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694848703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.694879159Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695149802Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_connection connected from Ok(127.0.0.1:36802)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695190256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_connection connected from Ok(127.0.0.1:46086)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695222812Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_connection connected from Ok(127.0.0.1:45515)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695255577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695749799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695779425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695794513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.695808441Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.696037309Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.696068535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.69646631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.696494096Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.696652164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697098653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697138567Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697165154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697179661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697193719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697207118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697219306Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697232674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[0]R flush_numbers[0..12]: [201, 201, 201, 201, 201, 201, 196, 201, 196, 194, 193, 200]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697246552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[0]R generation[0..12]: [111, 111, 111, 111, 111, 111, 108, 111, 108, 107, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.69726061Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697274848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[1]R flush_numbers[0..12]: [201, 201, 201, 201, 201, 201, 196, 201, 196, 194, 193, 200]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697288936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[1]R generation[0..12]: [111, 111, 111, 111, 111, 111, 108, 111, 108, 107, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697302775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697316693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[2]R flush_numbers[0..12]: [201, 201, 201, 201, 201, 201, 196, 201, 196, 194, 193, 200]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697330161Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"[2]R generation[0..12]: [111, 111, 111, 111, 111, 111, 108, 111, 108, 107, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697343409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697368515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697393172Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697417299Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697441155Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697465292Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"":"downstairs","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"region info: 2024-09-27T03:54:30.697490638Z" block size: 4096 bytes ,"hostname":" sub_volume 0 blocks / extent: 20 w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697519414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"0","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697539542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"client":"1","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 ,"time":"2024-09-27T03:54:30.69755336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA total size: 2.3 MiB " encryption: no ,"pid":1629,"client":"2","":"downstairs","session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697570027Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697582686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"5c8ad893-45f4-422c-b920-b2021fd3d4f1 is now active with session: 8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697595594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697607572Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629,"session_id":"8f78fbac-89c7-4b2b-9e4d-fc17b3d2d398"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:30.697620201Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 286 len 16384 data: 11 10 9 7 02/30 Write block 187 len 4096 data: 7 03/30 Flush 04/30 Write block 329 len 36864 data: 10 10 8 9 10 11 10 9 10 05/30 Read block 582 len 40960 06/30 Read block 350 len 20480 07/30 Flush 08/30 Write block 124 len 36864 data: 9 10 12 12 12 12 13 13 12 09/30 Read block 265 len 12288 10/30 Read block 571 len 16384 11/30 Write block 489 len 28672 data: 7 8 7 6 6 5 4 12/30 Read block 503 len 24576 13/30 Read block 390 len 40960 14/30 Write block 209 len 24576 data: 10 10 8 7 8 8 15/30 Write block 125 len 4096 data: 11 16/30 Read block 536 len 8192 17/30 Read block 132 len 40960 18/30 Write block 275 len 8192 data: 12 12 19/30 Read block 38 len 8192 20/30 Read block 92 len 20480 21/30 Write block 408 len 16384 data: 13 14 11 10 22/30 Read block 541 len 36864 23/30 Read block 298 len 32768 24/30 Write block 225 len 28672 data: 10 13 14 15 14 13 10 25/30 Read block 376 len 16384 26/30 Write block 401 len 40960 data: 4 6 8 9 11 11 11 14 15 12 27/30 Read block 27 len 8192 28/30 Write block 233 len 16384 data: 6 8 6 7 29/30 Write block 67 len 4096 data: 14 30/30 Read block 221 len 8192 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 8 Acked 1007 Write 9 Done Done Done false 9 Acked 1008 Read 3 Done Done Done false 10 Acked 1009 Read 4 Done Done Done false 11 Acked 1010 Write 7 Done Done Done false 12 Acked 1011 Read 6 Done Done Done false 13 Acked 1012 Read 10 Done Done Done false 14 Acked 1013 Write 6 Done Done Done false 15 Acked 1014 Write 1 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:30.720609185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1629} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 201 201 201 F F F 1 020-039 111 111 111 201 201 201 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 111 111 111 201 201 201 T T T 4 080-099 111 111 111 201 201 201 F F F 5 100-119 111 111 111 201 201 201 F F F 6 120-139 108 108 108 196 196 196 T T T 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 112 112 112 202 202 202 F F F 10 200-219 106 106 106 193 193 193 T T T 11 220-239 110 110 110 200 200 200 T T T 12 240-259 111 111 111 201 201 201 F F F 13 260-279 108 108 108 196 196 196 T T T 14 280-299 112 112 112 202 202 202 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 112 112 112 203 203 203 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 111 111 111 201 201 201 F F F 19 380-399 108 108 108 195 195 195 F F F 20 400-419 106 106 106 193 193 193 T T T 21 420-439 106 106 106 192 192 192 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 111 111 111 201 201 201 F F F 24 480-499 104 104 104 191 191 191 T T T 25 500-519 106 106 106 192 192 192 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 112, Max flush: 203 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 201 201 201 F F F 1 020-039 111 111 111 201 201 201 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 113 113 113 204 204 204 F F F 4 080-099 111 111 111 201 201 201 F F F 5 100-119 111 111 111 201 201 201 F F F 6 120-139 113 113 113 204 204 204 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 112 112 112 202 202 202 F F F 10 200-219 113 113 113 204 204 204 F F F 11 220-239 113 113 113 204 204 204 F F F 12 240-259 111 111 111 201 201 201 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 112 112 112 202 202 202 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 112 112 112 203 203 203 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 111 111 111 201 201 201 F F F 19 380-399 108 108 108 195 195 195 F F F 20 400-419 113 113 113 204 204 204 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 111 111 111 201 201 201 F F F 24 480-499 113 113 113 204 204 204 F F F 25 500-519 106 106 106 192 192 192 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 113, Max flush: 204 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.062715252Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:35.106418025Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.106969449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.139853476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.140721216Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.140773009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.140791527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.140809354Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637} {"msg":"Crucible 2e79fea2-d598-4f01-b21f-1535460806c4 has session id: 3ed73cd5-7524-4874-9102-49226c9f5105","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.141332282Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Upstairs opts: Upstairs UUID: 2e79fea2-d598-4f01-b21f-1535460806c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.141366367Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Crucible stats registered with UUID: 2e79fea2-d598-4f01-b21f-1535460806c4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.141382945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173253553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637} {"msg":"2e79fea2-d598-4f01-b21f-1535460806c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173302516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173321353Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173352929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173370606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173387584Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173404302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173437217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173475662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173507778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173538214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_connection connected from Ok(127.0.0.1:38723)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173570449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_connection connected from Ok(127.0.0.1:60522)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173845511Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_connection connected from Ok(127.0.0.1:47503)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.173904963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.174297258Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.174338493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.174470495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.174498081Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.17457439Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.174593028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.17500776Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.175202784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.17536939Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.175539077Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.175559244Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.175573322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.175675578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.175693556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.175707234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176002703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.17602259Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[0]R flush_numbers[0..12]: [201, 201, 201, 204, 201, 201, 204, 201, 196, 202, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176043917Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[0]R generation[0..12]: [111, 111, 111, 113, 111, 111, 113, 111, 108, 112, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176058555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176072953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[1]R flush_numbers[0..12]: [201, 201, 201, 204, 201, 201, 204, 201, 196, 202, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176086801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[1]R generation[0..12]: [111, 111, 111, 113, 111, 111, 113, 111, 108, 112, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176100409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176114168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[2]R flush_numbers[0..12]: [201, 201, 201, 204, 201, 201, 204, 201, 196, 202, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176131125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[2]R generation[0..12]: [111, 111, 111, 113, 111, 111, 113, 111, 108, 112, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176145033Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176158772Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.17617179Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176185358Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176198196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176211124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"":"downstairs","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176224112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:54:35.176235991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"0","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176251988Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"1","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176267037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"client":"2","":"downstairs","session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176282554Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176297162Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"2e79fea2-d598-4f01-b21f-1535460806c4 is now active with session: 3ed73cd5-7524-4874-9102-49226c9f5105","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.17631232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176326728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637,"session_id":"3ed73cd5-7524-4874-9102-49226c9f5105"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:35.176341236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 406 len 24576 data: 12 12 15 16 13 11 02/30 Write block 317 len 24576 data: 7 6 8 6 8 10 03/30 Read block 143 len 40960 04/30 Read block 189 len 12288 05/30 Read block 306 len 4096 06/30 Read block 425 len 28672 07/30 Read block 264 len 40960 08/30 Read block 47 len 36864 09/30 Read block 582 len 36864 10/30 Read block 290 len 20480 11/30 Write block 375 len 36864 data: 10 14 17 12 15 13 10 11 11 12/30 Read block 40 len 4096 13/30 Flush 14/30 Write block 230 len 12288 data: 14 11 7 15/30 Write block 97 len 8192 data: 6 8 16/30 Flush 17/30 Flush 18/30 Write block 246 len 4096 data: 17 19/30 Write block 111 len 8192 data: 6 6 20/30 Write block 319 len 32768 data: 9 7 9 11 11 12 12 12 21/30 Read block 542 len 12288 22/30 Write block 479 len 28672 data: 10 11 10 11 12 11 11 23/30 Read block 465 len 28672 24/30 Write block 512 len 4096 data: 7 25/30 Read block 221 len 16384 26/30 Write block 335 len 12288 data: 11 10 11 27/30 Write block 209 len 24576 data: 11 11 9 8 9 9 28/30 Read block 450 len 16384 29/30 Write block 293 len 24576 data: 10 8 7 10 10 10 30/30 Write block 412 len 8192 data: 9 8 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Write 8 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:35.19977023Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1637} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 201 201 201 F F F 1 020-039 111 111 111 201 201 201 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 113 113 113 204 204 204 F F F 4 080-099 114 114 114 206 206 206 F F F 5 100-119 111 111 111 201 201 201 T T T 6 120-139 113 113 113 204 204 204 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 112 112 112 202 202 202 F F F 10 200-219 113 113 113 204 204 204 T T T 11 220-239 114 114 114 206 206 206 F F F 12 240-259 111 111 111 201 201 201 T T T 13 260-279 113 113 113 204 204 204 F F F 14 280-299 112 112 112 202 202 202 T T T 15 300-319 114 114 114 205 205 205 T T T 16 320-339 114 114 114 205 205 205 T T T 17 340-359 110 110 110 199 199 199 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 114 114 114 205 205 205 F F F 20 400-419 114 114 114 205 205 205 T T T 21 420-439 106 106 106 192 192 192 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 111 111 111 201 201 201 T T T 24 480-499 113 113 113 204 204 204 T T T 25 500-519 106 106 106 192 192 192 T T T 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 114, Max flush: 206 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 201 201 201 F F F 1 020-039 111 111 111 201 201 201 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 113 113 113 204 204 204 F F F 4 080-099 114 114 114 206 206 206 F F F 5 100-119 115 115 115 207 207 207 F F F 6 120-139 113 113 113 204 204 204 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 112 112 112 202 202 202 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 114 114 114 206 206 206 F F F 12 240-259 115 115 115 207 207 207 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 115 115 115 207 207 207 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 114 114 114 205 205 205 F F F 20 400-419 115 115 115 207 207 207 F F F 21 420-439 106 106 106 192 192 192 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 115 115 115 207 207 207 F F F 24 480-499 115 115 115 207 207 207 F F F 25 500-519 115 115 115 207 207 207 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 102 102 102 189 189 189 F F F Max gen: 115, Max flush: 207 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.601969557Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:39.645886671Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.646267188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.678911739Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.679640808Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.679687462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.67970235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.679716928Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645} {"msg":"Crucible c471dc39-ddec-4d1d-a972-5a65db9625ed has session id: 9c696154-9803-43c5-9e8e-dcfc32787dea","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.680156077Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Upstairs opts: Upstairs UUID: c471dc39-ddec-4d1d-a972-5a65db9625ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.680202071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Crucible stats registered with UUID: c471dc39-ddec-4d1d-a972-5a65db9625ed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.680232587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.710958322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645} {"msg":"c471dc39-ddec-4d1d-a972-5a65db9625ed active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.710988048Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711005296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711033142Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711059608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711092444Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711126009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711162544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711196419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711231734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.71126551Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_connection connected from Ok(127.0.0.1:53019)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711300315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_connection connected from Ok(127.0.0.1:45875)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711352528Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_connection connected from Ok(127.0.0.1:62901)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.711400041Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712029114Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712056951Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712181653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.71220481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712224347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712250584Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712741116Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712769512Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.712880847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713346962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713384857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713410604Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713429281Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713444249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713458117Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713486333Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713500691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[0]R flush_numbers[0..12]: [201, 201, 201, 204, 206, 207, 204, 201, 196, 202, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713563123Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[0]R generation[0..12]: [111, 111, 111, 113, 114, 115, 113, 111, 108, 112, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.71358312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713597658Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[1]R flush_numbers[0..12]: [201, 201, 201, 204, 206, 207, 204, 201, 196, 202, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713611526Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[1]R generation[0..12]: [111, 111, 111, 113, 114, 115, 113, 111, 108, 112, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713625304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713639132Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[2]R flush_numbers[0..12]: [201, 201, 201, 204, 206, 207, 204, 201, 196, 202, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713656579Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[2]R generation[0..12]: [111, 111, 111, 113, 114, 115, 113, 111, 108, 112, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713670777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713684796Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713697654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713711542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:54:39.7137248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713742598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"":"downstairs","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713755716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713767534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"0","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713780912Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"1","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713794011Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"client":"2","":"downstairs","session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713807089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713819217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"c471dc39-ddec-4d1d-a972-5a65db9625ed is now active with session: 9c696154-9803-43c5-9e8e-dcfc32787dea","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713831955Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"region info: Set Active after no reconciliation block size: 4096 bytes ","v":0," sub_volume 0 blocks / extent: 20 name":"crucible","level":30,"time":"2024-09-27T03:54:39.713849073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645,"session_id":"9c696154-9803-43c5-9e8e-dcfc32787dea"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:39.713864051Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 585 len 16384 data: 9 8 5 6 02/30 Read block 382 len 36864 03/30 Write block 434 len 8192 data: 4 4 04/30 Write block 83 len 32768 data: 8 8 10 9 9 11 10 10 05/30 Read block 126 len 32768 06/30 Write block 410 len 40960 data: 14 12 10 9 10 8 8 9 11 10 07/30 Write block 66 len 28672 data: 11 15 13 15 14 15 15 08/30 Read block 532 len 16384 09/30 Write block 59 len 36864 data: 6 6 7 7 8 9 11 12 16 10/30 Write block 260 len 12288 data: 6 9 8 11/30 Write block 117 len 8192 data: 11 10 12/30 Read block 79 len 28672 13/30 Read block 377 len 16384 14/30 Read block 66 len 32768 15/30 Read block 222 len 8192 16/30 Read block 33 len 24576 17/30 Read block 32 len 16384 18/30 Read block 359 len 28672 19/30 Read block 110 len 16384 20/30 Flush 21/30 Write block 290 len 4096 data: 7 22/30 Read block 238 len 32768 23/30 Read block 415 len 24576 24/30 Write block 586 len 28672 data: 9 6 7 5 6 7 7 25/30 Read block 347 len 20480 26/30 Read block 207 len 36864 27/30 Read block 347 len 8192 28/30 Write block 34 len 40960 data: 8 5 6 8 9 9 10 12 12 13 29/30 Read block 335 len 8192 30/30 Write block 247 len 40960 data: 19 18 18 14 12 11 8 8 6 7 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:39.78741924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1645} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 201 201 201 F F F 1 020-039 111 111 111 201 201 201 T T T 2 040-059 116 116 116 208 208 208 T T T 3 060-079 116 116 116 208 208 208 F F F 4 080-099 116 116 116 208 208 208 F F F 5 100-119 116 116 116 208 208 208 F F F 6 120-139 113 113 113 204 204 204 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 112 112 112 202 202 202 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 114 114 114 206 206 206 F F F 12 240-259 115 115 115 207 207 207 T T T 13 260-279 116 116 116 208 208 208 F F F 14 280-299 115 115 115 207 207 207 T T T 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 114 114 114 205 205 205 F F F 20 400-419 116 116 116 208 208 208 F F F 21 420-439 116 116 116 208 208 208 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 115 115 115 207 207 207 F F F 24 480-499 115 115 115 207 207 207 F F F 25 500-519 115 115 115 207 207 207 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 116 116 116 208 208 208 T T T Max gen: 116, Max flush: 208 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 201 201 201 F F F 1 020-039 117 117 117 209 209 209 F F F 2 040-059 117 117 117 209 209 209 F F F 3 060-079 116 116 116 208 208 208 F F F 4 080-099 116 116 116 208 208 208 F F F 5 100-119 116 116 116 208 208 208 F F F 6 120-139 113 113 113 204 204 204 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 112 112 112 202 202 202 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 114 114 114 206 206 206 F F F 12 240-259 117 117 117 209 209 209 F F F 13 260-279 116 116 116 208 208 208 F F F 14 280-299 117 117 117 209 209 209 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 114 114 114 205 205 205 F F F 20 400-419 116 116 116 208 208 208 F F F 21 420-439 116 116 116 208 208 208 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 115 115 115 207 207 207 F F F 24 480-499 115 115 115 207 207 207 F F F 25 500-519 115 115 115 207 207 207 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 117 117 117 209 209 209 F F F Max gen: 117, Max flush: 209 Begin loop 59 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.121120919Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.165193871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.165822144Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.165857959Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.165877036Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.165891814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653} {"msg":"Crucible e57b8f6a-9117-4bae-8ead-484da27a881c has session id: 5e1e564a-32df-450e-8f4b-120803e15075","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.166370338Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Upstairs opts: Upstairs UUID: e57b8f6a-9117-4bae-8ead-484da27a881c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.166404923Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Crucible stats registered with UUID: e57b8f6a-9117-4bae-8ead-484da27a881c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.166421671Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197469052Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653} {"msg":"e57b8f6a-9117-4bae-8ead-484da27a881c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197507437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197531594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197576698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197610513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197646068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197680303Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197715569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197749974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197783589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197824063Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_connection connected from Ok(127.0.0.1:48580)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197857839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_connection connected from Ok(127.0.0.1:37449)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197890074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_connection connected from Ok(127.0.0.1:36094)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.197971273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.198544514Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.1985728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.198589348Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.198613864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.198639161Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.198667597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.199281372Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.199322307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.199353662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.199940251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.199981705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200008672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.20002292Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200036998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200050406Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200062624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200075903Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[0]R flush_numbers[0..12]: [201, 209, 209, 208, 208, 208, 204, 201, 196, 202, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200089681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[0]R generation[0..12]: [111, 117, 117, 116, 116, 116, 113, 111, 108, 112, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200103499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200117477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[1]R flush_numbers[0..12]: [201, 209, 209, 208, 208, 208, 204, 201, 196, 202, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200131335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[1]R generation[0..12]: [111, 117, 117, 116, 116, 116, 113, 111, 108, 112, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200145003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200158841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[2]R flush_numbers[0..12]: [201, 209, 209, 208, 208, 208, 204, 201, 196, 202, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200172459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[2]R generation[0..12]: [111, 117, 117, 116, 116, 116, 113, 111, 108, 112, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200186147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:54:44.200202825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200220492Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200244279Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200266236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200280594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"":"downstairs","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200293892Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200305921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"0","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200323778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"1","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200348035Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"client":"2","":"downstairs","session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200363003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200375321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"e57b8f6a-9117-4bae-8ead-484da27a881c is now active with session: 5e1e564a-32df-450e-8f4b-120803e15075","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200388249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200408157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653,"session_id":"5e1e564a-32df-450e-8f4b-120803e15075"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:44.200428334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 16384 data: 9 11 12 14 02/30 Read block 270 len 20480 03/30 Read block 551 len 4096 04/30 Write block 418 len 16384 data: 12 11 9 8 05/30 Read block 487 len 12288 06/30 Read block 443 len 4096 07/30 Read block 372 len 32768 08/30 Read block 208 len 16384 09/30 Write block 63 len 40960 data: 9 10 12 13 17 14 16 15 16 16 10/30 Read block 305 len 36864 11/30 Write block 477 len 24576 data: 8 8 11 12 11 12 12/30 Write block 14 len 8192 data: 10 12 13/30 Read block 468 len 32768 14/30 Read block 575 len 40960 15/30 Write block 188 len 28672 data: 6 6 9 8 8 8 10 16/30 Read block 19 len 36864 17/30 Read block 149 len 16384 18/30 Flush 19/30 Write block 61 len 12288 data: 8 8 10 20/30 Write block 99 len 12288 data: 10 10 8 21/30 Read block 151 len 24576 22/30 Read block 96 len 20480 23/30 Read block 189 len 12288 24/30 Read block 129 len 28672 25/30 Write block 383 len 4096 data: 12 26/30 Write block 255 len 36864 data: 7 8 7 7 6 7 10 9 8 27/30 Write block 36 len 4096 data: 7 28/30 Read block 458 len 12288 29/30 Read block 66 len 36864 30/30 Write block 22 len 12288 data: 11 8 6 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:44.222476188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1653} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 117 117 117 209 209 209 T T T 2 040-059 117 117 117 209 209 209 F F F 3 060-079 118 118 118 210 210 210 T T T 4 080-099 116 116 116 208 208 208 T T T 5 100-119 116 116 116 208 208 208 T T T 6 120-139 113 113 113 204 204 204 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 118 118 118 210 210 210 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 114 114 114 206 206 206 F F F 12 240-259 117 117 117 209 209 209 T T T 13 260-279 116 116 116 208 208 208 T T T 14 280-299 117 117 117 209 209 209 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 114 114 114 205 205 205 T T T 20 400-419 118 118 118 210 210 210 F F F 21 420-439 118 118 118 210 210 210 F F F 22 440-459 118 118 118 210 210 210 F F F 23 460-479 118 118 118 210 210 210 F F F 24 480-499 118 118 118 210 210 210 F F F 25 500-519 115 115 115 207 207 207 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 117 117 117 209 209 209 F F F Max gen: 118, Max flush: 210 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 119 119 119 211 211 211 F F F 2 040-059 117 117 117 209 209 209 F F F 3 060-079 119 119 119 211 211 211 F F F 4 080-099 119 119 119 211 211 211 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 113 113 113 204 204 204 F F F 7 140-159 111 111 111 201 201 201 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 118 118 118 210 210 210 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 114 114 114 206 206 206 F F F 12 240-259 119 119 119 211 211 211 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 117 117 117 209 209 209 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 110 110 110 199 199 199 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 119 119 119 211 211 211 F F F 20 400-419 118 118 118 210 210 210 F F F 21 420-439 118 118 118 210 210 210 F F F 22 440-459 118 118 118 210 210 210 F F F 23 460-479 118 118 118 210 210 210 F F F 24 480-499 118 118 118 210 210 210 F F F 25 500-519 115 115 115 207 207 207 F F F 26 520-539 108 108 108 197 197 197 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 107 107 107 194 194 194 F F F 29 580-599 117 117 117 209 209 209 F F F Max gen: 119, Max flush: 211 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.548561769Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.593310357Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.594171589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.594204834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.594219152Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.5942339Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661} {"msg":"Crucible 692b9ec9-b9e4-4c17-81d6-fa0d202d3c7d has session id: 1c175826-23bd-4d89-8d29-64d13933f9f0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.594716803Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Upstairs opts: Upstairs UUID: 692b9ec9-b9e4-4c17-81d6-fa0d202d3c7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.594763227Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Crucible stats registered with UUID: 692b9ec9-b9e4-4c17-81d6-fa0d202d3c7d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.594783394Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626337035Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661} {"msg":"692b9ec9-b9e4-4c17-81d6-fa0d202d3c7d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626398777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626432052Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626478946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626501583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626532598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626569593Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626604628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626637994Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626672649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626707454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46779)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626744209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63922)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626869612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39806)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.626911256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.627514813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.627554817Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.627585263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.627629397Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.627792934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.62782108Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.62826018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.62840242Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.628423337Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.628862686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.628891982Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.628916669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.628941315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.628969982Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.628994868Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629015335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629030273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[0]R flush_numbers[0..12]: [210, 211, 209, 211, 211, 211, 204, 201, 196, 210, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629095734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[0]R generation[0..12]: [118, 119, 117, 119, 119, 119, 113, 111, 108, 118, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629116122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.62913042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[1]R flush_numbers[0..12]: [210, 211, 209, 211, 211, 211, 204, 201, 196, 210, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629144438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[1]R generation[0..12]: [118, 119, 117, 119, 119, 119, 113, 111, 108, 118, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629158176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629172214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[2]R flush_numbers[0..12]: [210, 211, 209, 211, 211, 211, 204, 201, 196, 210, 207, 206]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629186032Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[2]R generation[0..12]: [118, 119, 117, 119, 119, 119, 113, 111, 108, 118, 115, 114]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.62919962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629213458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629226336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629243124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629256472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.62926957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629289527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629301666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"0","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629315154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"1","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629328452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"client":"2","":"downstairs","session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.62934183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629354089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"692b9ec9-b9e4-4c17-81d6-fa0d202d3c7d is now active with session: 1c175826-23bd-4d89-8d29-64d13933f9f0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629367097Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629379165Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661,"session_id":"1c175826-23bd-4d89-8d29-64d13933f9f0"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:48.629391233Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 127 len 28672 data: 13 13 13 14 14 13 11 02/30 Write block 577 len 16384 data: 5 5 6 5 03/30 Read block 216 len 36864 04/30 Read block 193 len 8192 05/30 Read block 50 len 32768 06/30 Read block 33 len 28672 07/30 Write block 257 len 4096 data: 8 08/30 Flush 09/30 Write block 342 len 16384 data: 8 6 8 7 10/30 Read block 576 len 12288 11/30 Read block 515 len 20480 12/30 Write block 239 len 40960 data: 8 7 6 6 9 11 12 18 20 19 13/30 Write block 39 len 20480 data: 10 11 13 13 14 14/30 Write block 70 len 16384 data: 16 17 17 16 15/30 Flush 16/30 Write block 150 len 8192 data: 7 8 17/30 Write block 397 len 20480 data: 6 6 4 4 5 18/30 Read block 501 len 32768 19/30 Write block 240 len 4096 data: 8 20/30 Read block 551 len 40960 21/30 Read block 202 len 4096 22/30 Read block 348 len 28672 23/30 Write block 516 len 36864 data: 12 12 10 10 10 9 8 9 10 24/30 Write block 190 len 32768 data: 10 9 9 9 11 10 10 7 25/30 Write block 159 len 4096 data: 9 26/30 Read block 537 len 40960 27/30 Read block 167 len 32768 28/30 Write block 46 len 20480 data: 15 13 13 15 13 29/30 Read block 248 len 36864 30/30 Read block 291 len 36864 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 2 Done Done Done false 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Read 8 Done Done Done false 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:48.652750317Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1661} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 120 120 120 213 213 213 F F F 2 040-059 120 120 120 213 213 213 T T T 3 060-079 120 120 120 213 213 213 F F F 4 080-099 119 119 119 211 211 211 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 120 120 120 212 212 212 F F F 7 140-159 111 111 111 201 201 201 T T T 8 160-179 108 108 108 196 196 196 F F F 9 180-199 118 118 118 210 210 210 T T T 10 200-219 115 115 115 207 207 207 F F F 11 220-239 120 120 120 213 213 213 F F F 12 240-259 120 120 120 213 213 213 T T T 13 260-279 119 119 119 211 211 211 F F F 14 280-299 117 117 117 209 209 209 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 119 119 119 211 211 211 T T T 20 400-419 118 118 118 210 210 210 T T T 21 420-439 118 118 118 210 210 210 F F F 22 440-459 118 118 118 210 210 210 F F F 23 460-479 118 118 118 210 210 210 F F F 24 480-499 118 118 118 210 210 210 F F F 25 500-519 115 115 115 207 207 207 T T T 26 520-539 108 108 108 197 197 197 T T T 27 540-559 108 108 108 197 197 197 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 120 120 120 212 212 212 F F F Max gen: 120, Max flush: 213 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 120 120 120 213 213 213 F F F 2 040-059 121 121 121 214 214 214 F F F 3 060-079 120 120 120 213 213 213 F F F 4 080-099 119 119 119 211 211 211 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 120 120 120 212 212 212 F F F 7 140-159 121 121 121 214 214 214 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 121 121 121 214 214 214 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 120 120 120 213 213 213 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 117 117 117 209 209 209 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 121 121 121 214 214 214 F F F 20 400-419 121 121 121 214 214 214 F F F 21 420-439 118 118 118 210 210 210 F F F 22 440-459 118 118 118 210 210 210 F F F 23 460-479 118 118 118 210 210 210 F F F 24 480-499 118 118 118 210 210 210 F F F 25 500-519 121 121 121 214 214 214 F F F 26 520-539 121 121 121 214 214 214 F F F 27 540-559 108 108 108 197 197 197 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 120 120 120 212 212 212 F F F Max gen: 121, Max flush: 214 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:52.994520751Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:53.038639806Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.039225185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.072734247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.073758575Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.07379574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.073811048Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.073831335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669} {"msg":"Crucible 1d091124-3b91-4ebf-85e3-7eabd4c18f40 has session id: 78d769cf-be9d-4ed1-b1c4-9efaab838388","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.07429824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Upstairs opts: Upstairs UUID: 1d091124-3b91-4ebf-85e3-7eabd4c18f40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.074351813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Crucible stats registered with UUID: 1d091124-3b91-4ebf-85e3-7eabd4c18f40","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.074382929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106328626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669} {"msg":"1d091124-3b91-4ebf-85e3-7eabd4c18f40 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106378079Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106396247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106424833Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.10644246Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106459788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106476615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106493723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106510181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106544696Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106580461Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_connection connected from Ok(127.0.0.1:44599)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106758967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_connection connected from Ok(127.0.0.1:52706)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106813529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_connection connected from Ok(127.0.0.1:65061)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.106848824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.107786734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.107825369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.10788862Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.107909048Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.107927515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.107951652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.108517074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.1085454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.108560648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.10912041Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109147986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109163115Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109230865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109250962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.10926477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109277249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109290707Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[0]R flush_numbers[0..12]: [210, 213, 214, 213, 211, 211, 212, 214, 196, 214, 207, 213]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109304815Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[0]R generation[0..12]: [118, 120, 121, 120, 119, 119, 120, 121, 108, 121, 115, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109318703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109332531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[1]R flush_numbers[0..12]: [210, 213, 214, 213, 211, 211, 212, 214, 196, 214, 207, 213]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109346409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[1]R generation[0..12]: [118, 120, 121, 120, 119, 119, 120, 121, 108, 121, 115, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109359997Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109373875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[2]R flush_numbers[0..12]: [210, 213, 214, 213, 211, 211, 212, 214, 196, 214, 207, 213]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109387624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[2]R generation[0..12]: [118, 120, 121, 120, 119, 119, 120, 121, 108, 121, 115, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109401282Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.10941519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109431757Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109445356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109458534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109471622Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"":"downstairs","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.10948487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109496948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"0","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:54:53.109510296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"1","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109526674Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"client":"2","":"downstairs","session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109540252Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109552541Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"1d091124-3b91-4ebf-85e3-7eabd4c18f40 is now active with session: 78d769cf-be9d-4ed1-b1c4-9efaab838388","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109565409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109577467Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669,"session_id":"78d769cf-be9d-4ed1-b1c4-9efaab838388"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:53.109589756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 513 len 12288 data: 8 11 13 02/30 Flush 03/30 Write block 576 len 16384 data: 4 6 6 7 04/30 Read block 307 len 12288 05/30 Read block 427 len 40960 06/30 Read block 83 len 28672 07/30 Read block 415 len 32768 08/30 Read block 555 len 24576 09/30 Write block 394 len 12288 data: 8 8 7 10/30 Flush 11/30 Write block 45 len 20480 data: 15 16 14 14 16 12/30 Write block 118 len 24576 data: 11 13 12 11 11 8 13/30 Read block 259 len 40960 14/30 Read block 327 len 20480 15/30 Read block 397 len 16384 16/30 Read block 125 len 4096 17/30 Read block 75 len 4096 18/30 Read block 439 len 40960 19/30 Flush 20/30 Write block 479 len 8192 data: 12 13 21/30 Flush 22/30 Write block 558 len 8192 data: 10 9 23/30 Read block 251 len 4096 24/30 Read block 195 len 32768 25/30 Read block 344 len 24576 26/30 Write block 436 len 36864 data: 5 5 5 4 4 5 5 4 4 27/30 Read block 571 len 12288 28/30 Read block 547 len 20480 29/30 Read block 371 len 4096 30/30 Read block 416 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:53.134623958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1669} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 120 120 120 213 213 213 F F F 2 040-059 122 122 122 217 217 217 F F F 3 060-079 120 120 120 213 213 213 F F F 4 080-099 119 119 119 211 211 211 F F F 5 100-119 122 122 122 217 217 217 F F F 6 120-139 122 122 122 217 217 217 F F F 7 140-159 121 121 121 214 214 214 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 121 121 121 214 214 214 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 120 120 120 213 213 213 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 117 117 117 209 209 209 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 122 122 122 216 216 216 F F F 20 400-419 121 121 121 214 214 214 F F F 21 420-439 118 118 118 210 210 210 T T T 22 440-459 118 118 118 210 210 210 T T T 23 460-479 122 122 122 218 218 218 F F F 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 121 121 121 214 214 214 F F F 27 540-559 108 108 108 197 197 197 T T T 28 560-579 122 122 122 216 216 216 F F F 29 580-599 120 120 120 212 212 212 F F F Max gen: 122, Max flush: 218 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 120 120 120 213 213 213 F F F 2 040-059 122 122 122 217 217 217 F F F 3 060-079 120 120 120 213 213 213 F F F 4 080-099 119 119 119 211 211 211 F F F 5 100-119 122 122 122 217 217 217 F F F 6 120-139 122 122 122 217 217 217 F F F 7 140-159 121 121 121 214 214 214 F F F 8 160-179 108 108 108 196 196 196 F F F 9 180-199 121 121 121 214 214 214 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 120 120 120 213 213 213 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 117 117 117 209 209 209 F F F 15 300-319 115 115 115 207 207 207 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 122 122 122 216 216 216 F F F 20 400-419 121 121 121 214 214 214 F F F 21 420-439 123 123 123 219 219 219 F F F 22 440-459 123 123 123 219 219 219 F F F 23 460-479 122 122 122 218 218 218 F F F 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 121 121 121 214 214 214 F F F 27 540-559 123 123 123 219 219 219 F F F 28 560-579 122 122 122 216 216 216 F F F 29 580-599 120 120 120 212 212 212 F F F Max gen: 123, Max flush: 219 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.455202349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:57.498704071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.499101416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.531097636Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.531773133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.531808918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.531823316Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.531837944Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677} {"msg":"Crucible ae064568-d752-417e-8ccd-7b5242d995f9 has session id: d07535b8-b5f4-4930-aa36-2aec56249a8b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.532287741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Upstairs opts: Upstairs UUID: ae064568-d752-417e-8ccd-7b5242d995f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.532332745Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Crucible stats registered with UUID: ae064568-d752-417e-8ccd-7b5242d995f9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.532352472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563368978Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677} {"msg":"ae064568-d752-417e-8ccd-7b5242d995f9 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563412732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563432609Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563462235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563485912Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563517927Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563550063Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563582669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563613314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.56364475Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563676646Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41547)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563708941Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58264)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.563747486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61398)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.564017009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.564490633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.56451802Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.564663219Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.564683677Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.564723731Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.564740299Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.565207894Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.56524078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.565577003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.565844706Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.565871553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.5658862Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.565944312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.565967399Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.565982037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566095082Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566113419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[0]R flush_numbers[0..12]: [210, 213, 217, 213, 211, 217, 217, 214, 196, 214, 207, 213]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566139085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[0]R generation[0..12]: [118, 120, 122, 120, 119, 122, 122, 121, 108, 121, 115, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566153434Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566167492Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[1]R flush_numbers[0..12]: [210, 213, 217, 213, 211, 217, 217, 214, 196, 214, 207, 213]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.5661811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[1]R generation[0..12]: [118, 120, 122, 120, 119, 122, 122, 121, 108, 121, 115, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566194768Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566208726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[2]R flush_numbers[0..12]: [210, 213, 217, 213, 211, 217, 217, 214, 196, 214, 207, 213]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566222494Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[2]R generation[0..12]: [118, 120, 122, 120, 119, 122, 122, 121, 108, 121, 115, 120]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566236112Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.56625Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566263058Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566280536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566294034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566307002Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"":"downstairs","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566320051Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-09-27T03:54:57.566338668Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"0","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566362325Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"1","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566387331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"client":"2","":"downstairs","session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566412268Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566434925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"ae064568-d752-417e-8ccd-7b5242d995f9 is now active with session: d07535b8-b5f4-4930-aa36-2aec56249a8b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566460021Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566484188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677,"session_id":"d07535b8-b5f4-4930-aa36-2aec56249a8b"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:54:57.566509924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 392 len 4096 data: 9 02/30 Read block 538 len 8192 03/30 Read block 204 len 36864 04/30 Write block 579 len 8192 data: 8 6 05/30 Write block 178 len 36864 data: 9 9 8 12 11 12 10 9 10 06/30 Read block 450 len 40960 07/30 Write block 126 len 36864 data: 13 14 14 14 15 15 14 12 11 08/30 Write block 83 len 28672 data: 9 9 11 10 10 12 11 09/30 Write block 280 len 24576 data: 9 9 7 8 8 10 10/30 Write block 220 len 12288 data: 7 11 12 11/30 Write block 67 len 12288 data: 18 15 17 12/30 Read block 298 len 12288 13/30 Read block 106 len 20480 14/30 Read block 163 len 4096 15/30 Read block 328 len 16384 16/30 Read block 369 len 8192 17/30 Write block 577 len 12288 data: 7 7 9 18/30 Write block 535 len 16384 data: 11 12 9 9 19/30 Write block 97 len 24576 data: 7 9 11 11 9 8 20/30 Read block 344 len 4096 21/30 Write block 177 len 32768 data: 9 10 10 9 13 12 13 11 22/30 Write block 116 len 24576 data: 12 12 12 14 13 12 23/30 Read block 346 len 32768 24/30 Read block 93 len 20480 25/30 Write block 463 len 12288 data: 9 9 12 26/30 Write block 525 len 8192 data: 10 9 27/30 Read block 135 len 40960 28/30 Write block 298 len 32768 data: 11 10 9 10 9 8 12 13 29/30 Read block 285 len 36864 30/30 Read block 457 len 20480 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 1 Done Done Done false 2 Acked 1001 Read 2 Done Done Done false 3 Acked 1002 Read 9 Done Done Done false 4 Acked 1003 Write 2 Done Done Done false 5 Acked 1004 Write 9 Done Done Done false 6 Acked 1005 Read 10 Done Done Done false 7 Acked 1006 Write 9 Done Done Done false 8 Acked 1007 Write 7 Done Done Done false 9 Acked 1008 Write 6 Done Done Done false 10 Acked 1009 Write 3 Done Done Done false 11 Acked 1010 Write 3 Done Done Done false 12 Acked 1011 Read 3 Done Done Done false 13 Acked 1012 Read 5 Done Done Done false 14 Acked 1013 Read 1 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:54:57.585421652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1677} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 120 120 120 213 213 213 F F F 2 040-059 122 122 122 217 217 217 F F F 3 060-079 120 120 120 213 213 213 T T T 4 080-099 119 119 119 211 211 211 T T T 5 100-119 122 122 122 217 217 217 T T T 6 120-139 122 122 122 217 217 217 T T T 7 140-159 121 121 121 214 214 214 F F F 8 160-179 108 108 108 196 196 196 T T T 9 180-199 121 121 121 214 214 214 T T T 10 200-219 115 115 115 207 207 207 F F F 11 220-239 120 120 120 213 213 213 T T T 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 117 117 117 209 209 209 T T T 15 300-319 115 115 115 207 207 207 T T T 16 320-339 115 115 115 207 207 207 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 122 122 122 216 216 216 T T T 20 400-419 121 121 121 214 214 214 F F F 21 420-439 123 123 123 219 219 219 F F F 22 440-459 123 123 123 219 219 219 F F F 23 460-479 122 122 122 218 218 218 T T T 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 121 121 121 214 214 214 T T T 27 540-559 123 123 123 219 219 219 F F F 28 560-579 122 122 122 216 216 216 T T T 29 580-599 120 120 120 212 212 212 T T T Max gen: 123, Max flush: 219 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 F F F 1 020-039 120 120 120 213 213 213 F F F 2 040-059 122 122 122 217 217 217 F F F 3 060-079 124 124 124 220 220 220 F F F 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 124 124 124 220 220 220 F F F 7 140-159 121 121 121 214 214 214 F F F 8 160-179 124 124 124 220 220 220 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 124 124 124 220 220 220 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 124 124 124 220 220 220 F F F 15 300-319 124 124 124 220 220 220 F F F 16 320-339 115 115 115 207 207 207 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 124 124 124 220 220 220 F F F 20 400-419 121 121 121 214 214 214 F F F 21 420-439 123 123 123 219 219 219 F F F 22 440-459 123 123 123 219 219 219 F F F 23 460-479 124 124 124 220 220 220 F F F 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 124 124 124 220 220 220 F F F 27 540-559 123 123 123 219 219 219 F F F 28 560-579 124 124 124 220 220 220 F F F 29 580-599 124 124 124 220 220 220 F F F Max gen: 124, Max flush: 220 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.039818168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.085185431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.086053071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.086089756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.086109574Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.086124531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685} {"msg":"Crucible 785c7772-98e8-4b6c-8b6d-acd62836edda has session id: e1f4313b-cea3-4049-8cdf-58bc1099b6c6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.086508938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Upstairs opts: Upstairs UUID: 785c7772-98e8-4b6c-8b6d-acd62836edda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.086550202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Crucible stats registered with UUID: 785c7772-98e8-4b6c-8b6d-acd62836edda","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.086574529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118684314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685} {"msg":"785c7772-98e8-4b6c-8b6d-acd62836edda active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118726838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118744695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118773351Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118802697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118842422Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118881776Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118919121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118955136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.118990492Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.119027736Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45506)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.119062211Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46267)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.119093587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45197)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.119122233Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.119808978Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.119845943Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.120103328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.120137693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.12015904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.120175388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.120552016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.120625146Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.120862803Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.12124561Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121282315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121312311Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121330938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121345256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121358844Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121375452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.12138885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[0]R flush_numbers[0..12]: [210, 213, 217, 220, 220, 220, 220, 214, 220, 220, 207, 220]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.12145814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[0]R generation[0..12]: [118, 120, 122, 124, 124, 124, 124, 121, 124, 124, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121477008Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121491556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[1]R flush_numbers[0..12]: [210, 213, 217, 220, 220, 220, 220, 214, 220, 220, 207, 220]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121505404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[1]R generation[0..12]: [118, 120, 122, 124, 124, 124, 124, 121, 124, 124, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121519212Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.12153298Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[2]R flush_numbers[0..12]: [210, 213, 217, 220, 220, 220, 220, 214, 220, 220, 207, 220]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121546658Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[2]R generation[0..12]: [118, 120, 122, 124, 124, 124, 124, 121, 124, 124, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121560346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121577344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Max found gen is 125","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121590682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Generation requested: 126 >= found:125","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.12160372Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121616629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121629607Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"":"downstairs","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"No downstairs reconciliation required","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-09-27T03:55:02.121649544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121661832Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"0","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121675071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"1","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121687969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"client":"2","":"downstairs","session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121701097Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121713295Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"785c7772-98e8-4b6c-8b6d-acd62836edda is now active with session: e1f4313b-cea3-4049-8cdf-58bc1099b6c6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121726143Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.121738242Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685,"session_id":"e1f4313b-cea3-4049-8cdf-58bc1099b6c6"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:02.12175044Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 8192 data: 8 9 02/30 Read block 20 len 8192 03/30 Write block 456 len 40960 data: 10 12 13 15 13 13 11 10 10 13 04/30 Read block 24 len 16384 05/30 Read block 381 len 16384 06/30 Write block 562 len 28672 data: 5 5 7 8 9 10 10 07/30 Read block 277 len 20480 08/30 Write block 168 len 28672 data: 4 4 3 3 3 4 7 09/30 Write block 316 len 36864 data: 6 8 7 10 8 10 12 12 13 10/30 Read block 334 len 20480 11/30 Read block 51 len 16384 12/30 Write block 573 len 20480 data: 10 8 6 5 8 13/30 Flush 14/30 Write block 544 len 40960 data: 10 9 10 8 7 6 4 4 4 8 15/30 Read block 206 len 12288 16/30 Read block 196 len 24576 17/30 Write block 175 len 4096 data: 6 18/30 Read block 282 len 36864 19/30 Read block 548 len 32768 20/30 Write block 399 len 36864 data: 5 5 6 7 9 10 12 13 13 21/30 Write block 525 len 24576 data: 11 10 8 11 11 13 22/30 Write block 64 len 32768 data: 11 13 14 19 16 18 17 18 23/30 Read block 302 len 24576 24/30 Write block 2 len 28672 data: 6 7 6 6 8 11 13 25/30 Read block 143 len 32768 26/30 Read block 159 len 16384 27/30 Read block 351 len 12288 28/30 Read block 153 len 8192 29/30 Read block 309 len 16384 30/30 Read block 591 len 12288 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 10 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 6 Done Done Done false 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 17 16 17 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:02.145233096Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1685} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 210 210 210 T T T 1 020-039 120 120 120 213 213 213 F F F 2 040-059 122 122 122 217 217 217 F F F 3 060-079 124 124 124 220 220 220 T T T 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 124 124 124 220 220 220 F F F 7 140-159 121 121 121 214 214 214 F F F 8 160-179 126 126 126 221 221 221 T T T 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 124 124 124 220 220 220 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 124 124 124 220 220 220 F F F 15 300-319 126 126 126 221 221 221 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 124 124 124 220 220 220 T T T 20 400-419 121 121 121 214 214 214 T T T 21 420-439 123 123 123 219 219 219 F F F 22 440-459 126 126 126 221 221 221 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 124 124 124 220 220 220 T T T 27 540-559 123 123 123 219 219 219 T T T 28 560-579 126 126 126 221 221 221 F F F 29 580-599 124 124 124 220 220 220 F F F Max gen: 126, Max flush: 221 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 222 222 222 F F F 1 020-039 120 120 120 213 213 213 F F F 2 040-059 122 122 122 217 217 217 F F F 3 060-079 127 127 127 222 222 222 F F F 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 124 124 124 220 220 220 F F F 7 140-159 121 121 121 214 214 214 F F F 8 160-179 127 127 127 222 222 222 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 124 124 124 220 220 220 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 124 124 124 220 220 220 F F F 15 300-319 126 126 126 221 221 221 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 114 114 114 205 205 205 F F F 19 380-399 127 127 127 222 222 222 F F F 20 400-419 127 127 127 222 222 222 F F F 21 420-439 123 123 123 219 219 219 F F F 22 440-459 126 126 126 221 221 221 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 127 127 127 222 222 222 F F F 28 560-579 126 126 126 221 221 221 F F F 29 580-599 124 124 124 220 220 220 F F F Max gen: 127, Max flush: 222 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.487111846Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.532087893Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.532963542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.533008036Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.533029593Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.533045101Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693} {"msg":"Crucible 28f14e2c-be3c-471e-925d-f6e2a338d606 has session id: 27b79229-b964-472b-b322-209508c78835","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.533526555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Upstairs opts: Upstairs UUID: 28f14e2c-be3c-471e-925d-f6e2a338d606, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.53355826Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Crucible stats registered with UUID: 28f14e2c-be3c-471e-925d-f6e2a338d606","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.533573968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565057539Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693} {"msg":"28f14e2c-be3c-471e-925d-f6e2a338d606 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565113001Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565145427Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.56519249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565229515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565263911Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565347559Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565390313Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565423149Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565456794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565585056Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_connection connected from Ok(127.0.0.1:40873)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.56562919Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_connection connected from Ok(127.0.0.1:49020)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.565667585Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_connection connected from Ok(127.0.0.1:65510)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.56569947Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566296408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566333293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566397824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566433439Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566450437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566464635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566910423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.566953287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567119454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567568362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567594958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567613486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567692315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567711053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567724751Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567737019Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567750367Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[0]R flush_numbers[0..12]: [222, 213, 217, 222, 220, 220, 220, 214, 222, 220, 207, 220]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567807909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[0]R generation[0..12]: [127, 120, 122, 127, 124, 124, 124, 121, 127, 124, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567826666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567840864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[1]R flush_numbers[0..12]: [222, 213, 217, 222, 220, 220, 220, 214, 222, 220, 207, 220]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567854643Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[1]R generation[0..12]: [127, 120, 122, 127, 124, 124, 124, 121, 127, 124, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567868501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567882509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[2]R flush_numbers[0..12]: [222, 213, 217, 222, 220, 220, 220, 214, 222, 220, 207, 220]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567896257Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[2]R generation[0..12]: [127, 120, 122, 127, 124, 124, 124, 121, 127, 124, 115, 124]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567910055Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567927153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567940581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567954129Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.567967097Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:55:06.567987364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"":"downstairs","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568008731Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.56802156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"0","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568034908Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"1","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568048076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"client":"2","":"downstairs","session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568061284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568073512Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"28f14e2c-be3c-471e-925d-f6e2a338d606 is now active with session: 27b79229-b964-472b-b322-209508c78835","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568086401Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568098379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693,"session_id":"27b79229-b964-472b-b322-209508c78835"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:06.568110567Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 24576 data: 11 12 9 7 7 9 02/30 Flush 03/30 Write block 457 len 40960 data: 13 14 16 14 14 12 11 11 14 11 04/30 Write block 361 len 20480 data: 12 11 11 10 9 05/30 Read block 102 len 28672 06/30 Flush 07/30 Write block 122 len 16384 data: 12 9 10 12 08/30 Write block 560 len 8192 data: 8 7 09/30 Read block 30 len 32768 10/30 Read block 474 len 40960 11/30 Read block 537 len 24576 12/30 Read block 489 len 36864 13/30 Write block 299 len 16384 data: 11 10 11 10 14/30 Write block 169 len 12288 data: 5 4 4 15/30 Write block 360 len 36864 data: 12 13 12 12 11 10 10 12 14 16/30 Write block 33 len 28672 data: 9 9 6 8 9 10 11 17/30 Read block 539 len 12288 18/30 Read block 353 len 4096 19/30 Write block 449 len 16384 data: 8 6 6 7 20/30 Read block 449 len 24576 21/30 Write block 557 len 40960 data: 7 11 10 9 8 6 6 8 9 10 22/30 Read block 115 len 32768 23/30 Write block 220 len 36864 data: 8 12 13 9 11 11 14 15 16 24/30 Read block 327 len 36864 25/30 Write block 156 len 12288 data: 7 9 10 26/30 Read block 125 len 20480 27/30 Read block 164 len 8192 28/30 Write block 427 len 28672 data: 7 5 5 5 6 7 4 29/30 Read block 423 len 32768 30/30 Write block 7 len 8192 data: 12 14 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 7 Acked 1006 Write 4 Done Done Done false 8 Acked 1007 Write 2 Done Done Done false 9 Acked 1008 Read 8 Done Done Done false 10 Acked 1009 Read 10 Done Done Done false 11 Acked 1010 Read 6 Done Done Done false 12 Acked 1011 Read 9 Done Done Done false 13 Acked 1012 Write 4 Done Done Done false 14 Acked 1013 Write 3 Done Done Done false 15 Acked 1014 Write 9 Done Done Done false 16 Acked 1015 Write 7 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Read 1 Done Done Done false 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 10 Done Done Done false 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 8 Done Done Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 23 23 22 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:06.593585608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1693} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 222 222 222 T T T 1 020-039 120 120 120 213 213 213 T T T 2 040-059 128 128 128 223 223 223 F F F 3 060-079 128 128 128 223 223 223 F F F 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 124 124 124 220 220 220 T T T 7 140-159 121 121 121 214 214 214 T T T 8 160-179 127 127 127 222 222 222 T T T 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 124 124 124 220 220 220 T T T 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 124 124 124 220 220 220 T T T 15 300-319 126 126 126 221 221 221 T T T 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 128 128 128 224 224 224 T T T 19 380-399 127 127 127 222 222 222 F F F 20 400-419 127 127 127 222 222 222 F F F 21 420-439 123 123 123 219 219 219 T T T 22 440-459 128 128 128 224 224 224 T T T 23 460-479 128 128 128 224 224 224 F F F 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 127 127 127 222 222 222 T T T 28 560-579 126 126 126 221 221 221 T T T 29 580-599 124 124 124 220 220 220 F F F Max gen: 128, Max flush: 224 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 225 225 225 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 128 128 128 223 223 223 F F F 3 060-079 128 128 128 223 223 223 F F F 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 129 129 129 225 225 225 F F F 7 140-159 129 129 129 225 225 225 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 129 129 129 225 225 225 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 F F F 14 280-299 129 129 129 225 225 225 F F F 15 300-319 129 129 129 225 225 225 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 129 129 129 225 225 225 F F F 19 380-399 127 127 127 222 222 222 F F F 20 400-419 127 127 127 222 222 222 F F F 21 420-439 129 129 129 225 225 225 F F F 22 440-459 129 129 129 225 225 225 F F F 23 460-479 128 128 128 224 224 224 F F F 24 480-499 122 122 122 218 218 218 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 129 129 129 225 225 225 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 124 124 124 220 220 220 F F F Max gen: 129, Max flush: 225 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.029059598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.073719809Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.074600697Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.074642741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.074666648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.074685935Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701} {"msg":"Crucible a1905409-2b2f-4cc5-902e-7cbf671dfee6 has session id: 3a61578d-5409-4ee0-b76a-77c2c4068036","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.075134984Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Upstairs opts: Upstairs UUID: a1905409-2b2f-4cc5-902e-7cbf671dfee6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.075176418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Crucible stats registered with UUID: a1905409-2b2f-4cc5-902e-7cbf671dfee6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.075193875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.10708749Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701} {"msg":"a1905409-2b2f-4cc5-902e-7cbf671dfee6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107144782Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107176998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107225801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107284863Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107310709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107334686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107358322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107381639Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107411425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107437322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_connection connected from Ok(127.0.0.1:35437)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107462708Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_connection connected from Ok(127.0.0.1:48609)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107487115Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_connection connected from Ok(127.0.0.1:37929)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.107510611Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.108311731Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.108352555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.108384661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.108416336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.108448262Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.108471499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109057898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109082634Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109098702Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109614281Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109658755Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109688251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109753651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109780598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109795996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109808634Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109822452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[0]R flush_numbers[0..12]: [225, 225, 223, 223, 220, 220, 225, 225, 225, 220, 207, 225]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.10983638Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[0]R generation[0..12]: [129, 129, 128, 128, 124, 124, 129, 129, 129, 124, 115, 129]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109851848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109866086Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[1]R flush_numbers[0..12]: [225, 225, 223, 223, 220, 220, 225, 225, 225, 220, 207, 225]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109879914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[1]R generation[0..12]: [129, 129, 128, 128, 124, 124, 129, 129, 129, 124, 115, 129]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109893592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.10990735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[2]R flush_numbers[0..12]: [225, 225, 223, 223, 220, 220, 225, 225, 225, 220, 207, 225]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109921028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[2]R generation[0..12]: [129, 129, 128, 128, 124, 124, 129, 129, 129, 124, 115, 129]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109939076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109954444Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109967742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.10998126Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.109994368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110007467Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"":"downstairs","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110027754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110039882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"0","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.11005328Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"1","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110066528Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"client":"2","":"downstairs","session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110079886Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110092035Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"a1905409-2b2f-4cc5-902e-7cbf671dfee6 is now active with session: 3a61578d-5409-4ee0-b76a-77c2c4068036","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110104813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.110116691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701,"session_id":"3a61578d-5409-4ee0-b76a-77c2c4068036"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:11.11012883Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 463 len 24576 data: 12 12 15 12 10 8 02/30 Read block 108 len 28672 03/30 Write block 42 len 20480 data: 14 15 14 16 17 04/30 Write block 576 len 20480 data: 6 9 8 10 7 05/30 Read block 231 len 4096 06/30 Read block 132 len 32768 07/30 Read block 167 len 28672 08/30 Read block 436 len 4096 09/30 Write block 226 len 40960 data: 15 16 17 15 15 12 8 7 9 7 10/30 Read block 134 len 4096 11/30 Write block 448 len 28672 data: 8 9 7 7 8 8 8 12/30 Read block 579 len 12288 13/30 Write block 260 len 28672 data: 8 11 10 9 9 8 7 14/30 Write block 295 len 4096 data: 8 15/30 Write block 485 len 28672 data: 12 9 7 6 8 9 8 16/30 Read block 497 len 24576 17/30 Read block 123 len 24576 18/30 Write block 379 len 24576 data: 16 14 11 12 13 11 19/30 Read block 542 len 8192 20/30 Read block 309 len 24576 21/30 Write block 495 len 8192 data: 5 7 22/30 Read block 281 len 12288 23/30 Read block 523 len 36864 24/30 Write block 493 len 24576 data: 7 6 6 8 9 8 25/30 Read block 384 len 8192 26/30 Read block 475 len 4096 27/30 Read block 469 len 24576 28/30 Write block 68 len 24576 data: 17 19 18 19 18 17 29/30 Write block 556 len 32768 data: 11 8 12 11 10 9 7 7 30/30 Write block 456 len 40960 data: 11 14 15 17 15 15 13 13 13 16 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Read 7 Done Done Done false 3 Acked 1002 Write 5 Done Done Done false 4 Acked 1003 Write 5 Done Done Done false 5 Acked 1004 Read 1 Done Done Done false 6 Acked 1005 Read 8 Done Done Done false 7 Acked 1006 Read 7 Done Done Done false 8 Acked 1007 Read 1 Done Done Done false 9 Acked 1008 Write 10 Done Done Done false 10 Acked 1009 Read 1 Done Done Done false 11 Acked 1010 Write 7 Done Done Done false 12 Acked 1011 Read 3 Done Done Done false 13 Acked 1012 Write 7 Done Done Done false 14 Acked 1013 Write 1 Done Done Done false 15 Acked 1014 Write 7 Done Done Done false 16 Acked 1015 Read 6 Done Done Done false 17 Acked 1016 Read 6 Done Done Done false 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:11.130128247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1701} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 225 225 225 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 128 128 128 223 223 223 T T T 3 060-079 128 128 128 223 223 223 T T T 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 129 129 129 225 225 225 F F F 7 140-159 129 129 129 225 225 225 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 129 129 129 225 225 225 T T T 12 240-259 121 121 121 214 214 214 F F F 13 260-279 119 119 119 211 211 211 T T T 14 280-299 129 129 129 225 225 225 T T T 15 300-319 129 129 129 225 225 225 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 129 129 129 225 225 225 T T T 19 380-399 127 127 127 222 222 222 T T T 20 400-419 127 127 127 222 222 222 F F F 21 420-439 129 129 129 225 225 225 F F F 22 440-459 129 129 129 225 225 225 T T T 23 460-479 128 128 128 224 224 224 T T T 24 480-499 122 122 122 218 218 218 T T T 25 500-519 122 122 122 215 215 215 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 129 129 129 225 225 225 T T T 28 560-579 129 129 129 225 225 225 T T T 29 580-599 124 124 124 220 220 220 T T T Max gen: 129, Max flush: 225 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 225 225 225 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 130 130 130 226 226 226 F F F 3 060-079 130 130 130 226 226 226 F F F 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 129 129 129 225 225 225 F F F 7 140-159 129 129 129 225 225 225 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 F F F 11 220-239 130 130 130 226 226 226 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 130 130 130 226 226 226 F F F 14 280-299 130 130 130 226 226 226 F F F 15 300-319 129 129 129 225 225 225 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 130 130 130 226 226 226 F F F 19 380-399 130 130 130 226 226 226 F F F 20 400-419 127 127 127 222 222 222 F F F 21 420-439 129 129 129 225 225 225 F F F 22 440-459 130 130 130 226 226 226 F F F 23 460-479 130 130 130 226 226 226 F F F 24 480-499 130 130 130 226 226 226 F F F 25 500-519 122 122 122 215 215 215 F F F 26 520-539 127 127 127 222 222 222 F F F 27 540-559 130 130 130 226 226 226 F F F 28 560-579 130 130 130 226 226 226 F F F 29 580-599 130 130 130 226 226 226 F F F Max gen: 130, Max flush: 226 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.555406766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.599656424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.600490108Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.600533822Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.60054798Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.600562749Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709} {"msg":"Crucible 213cc27d-2b90-4cb8-bf17-7e7192acc2f1 has session id: 682b7bb9-8bf2-4387-b61f-73fcbc22a94a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.601024215Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Upstairs opts: Upstairs UUID: 213cc27d-2b90-4cb8-bf17-7e7192acc2f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.601093035Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Crucible stats registered with UUID: 213cc27d-2b90-4cb8-bf17-7e7192acc2f1","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.60112947Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633068468Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709} {"msg":"213cc27d-2b90-4cb8-bf17-7e7192acc2f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633097654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633114822Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633142718Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633160156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633177503Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.63319829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633259282Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633318134Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633356228Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633615483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40910)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633653487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43109)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.633677234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61744)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.63370525Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634048613Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634085288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634225159Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634245216Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634393235Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634460916Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634891907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.634944459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635078851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635476546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.63551558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635543497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635601618Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635629425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635644183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635668349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635682807Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[0]R flush_numbers[0..12]: [225, 225, 226, 226, 220, 220, 225, 225, 225, 220, 207, 226]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635744809Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[0]R generation[0..12]: [129, 129, 130, 130, 124, 124, 129, 129, 129, 124, 115, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635763666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635777884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[1]R flush_numbers[0..12]: [225, 225, 226, 226, 220, 220, 225, 225, 225, 220, 207, 226]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635791632Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[1]R generation[0..12]: [129, 129, 130, 130, 124, 124, 129, 129, 129, 124, 115, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.63580586Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635824907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[2]R flush_numbers[0..12]: [225, 225, 226, 226, 220, 220, 225, 225, 225, 220, 207, 226]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635839276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[2]R generation[0..12]: [129, 129, 130, 130, 124, 124, 129, 129, 129, 124, 115, 130]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635852954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635866882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Max found gen is 131","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.63587958Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Generation requested: 132 >= found:131","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635892838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635905606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:55:15.635918495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"":"downstairs","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635936612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635948601Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"0","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635962039Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"1","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635975307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"client":"2","":"downstairs","session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.635988515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.636000743Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"213cc27d-2b90-4cb8-bf17-7e7192acc2f1 is now active with session: 682b7bb9-8bf2-4387-b61f-73fcbc22a94a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.636013472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.636027599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709,"session_id":"682b7bb9-8bf2-4387-b61f-73fcbc22a94a"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:15.636039858Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 386 len 20480 data: 7 10 13 12 13 02/30 Write block 296 len 32768 data: 11 11 12 12 11 12 11 9 03/30 Read block 368 len 28672 04/30 Write block 138 len 36864 data: 11 13 11 11 9 10 9 11 11 05/30 Read block 214 len 20480 06/30 Read block 102 len 16384 07/30 Read block 499 len 8192 08/30 Write block 500 len 16384 data: 9 9 10 9 09/30 Read block 75 len 36864 10/30 Flush 11/30 Write block 218 len 20480 data: 7 7 9 13 14 12/30 Write block 469 len 16384 data: 8 9 10 10 13/30 Write block 519 len 36864 data: 11 11 10 9 10 11 12 11 9 14/30 Read block 72 len 20480 15/30 Read block 275 len 36864 16/30 Read block 487 len 28672 17/30 Read block 29 len 40960 18/30 Write block 306 len 4096 data: 12 19/30 Read block 238 len 4096 20/30 Write block 280 len 20480 data: 10 10 8 9 9 21/30 Read block 467 len 8192 22/30 Read block 356 len 24576 23/30 Read block 474 len 20480 24/30 Write block 360 len 20480 data: 13 14 13 13 12 25/30 Read block 454 len 16384 26/30 Read block 422 len 12288 27/30 Write block 132 len 12288 data: 15 13 12 28/30 Write block 578 len 16384 data: 9 11 8 7 29/30 Read block 153 len 40960 30/30 Write block 561 len 28672 data: 10 8 8 9 10 11 11 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 11 Acked 1010 Write 5 Done Done Done false 12 Acked 1011 Write 4 Done Done Done false 13 Acked 1012 Write 9 Done Done Done false 14 Acked 1013 Read 5 Done Done Done false 15 Acked 1014 Read 9 Done Done Done false 16 Acked 1015 Read 7 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Read 1 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:15.658699448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1709} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 225 225 225 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 130 130 130 226 226 226 F F F 3 060-079 130 130 130 226 226 226 F F F 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 132 132 132 227 227 227 T T T 7 140-159 132 132 132 227 227 227 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 115 115 115 207 207 207 T T T 11 220-239 130 130 130 226 226 226 T T T 12 240-259 121 121 121 214 214 214 F F F 13 260-279 130 130 130 226 226 226 F F F 14 280-299 132 132 132 227 227 227 T T T 15 300-319 132 132 132 227 227 227 T T T 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 130 130 130 226 226 226 T T T 19 380-399 132 132 132 227 227 227 F F F 20 400-419 127 127 127 222 222 222 F F F 21 420-439 129 129 129 225 225 225 F F F 22 440-459 130 130 130 226 226 226 F F F 23 460-479 130 130 130 226 226 226 T T T 24 480-499 130 130 130 226 226 226 F F F 25 500-519 132 132 132 227 227 227 T T T 26 520-539 127 127 127 222 222 222 T T T 27 540-559 130 130 130 226 226 226 F F F 28 560-579 130 130 130 226 226 226 T T T 29 580-599 130 130 130 226 226 226 T T T Max gen: 132, Max flush: 227 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 225 225 225 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 130 130 130 226 226 226 F F F 3 060-079 130 130 130 226 226 226 F F F 4 080-099 124 124 124 220 220 220 F F F 5 100-119 124 124 124 220 220 220 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 132 132 132 227 227 227 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 121 121 121 214 214 214 F F F 13 260-279 130 130 130 226 226 226 F F F 14 280-299 133 133 133 228 228 228 F F F 15 300-319 133 133 133 228 228 228 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 126 126 126 221 221 221 F F F 18 360-379 133 133 133 228 228 228 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 127 127 127 222 222 222 F F F 21 420-439 129 129 129 225 225 225 F F F 22 440-459 130 130 130 226 226 226 F F F 23 460-479 133 133 133 228 228 228 F F F 24 480-499 130 130 130 226 226 226 F F F 25 500-519 133 133 133 228 228 228 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 130 130 130 226 226 226 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 133 133 133 228 228 228 F F F Max gen: 133, Max flush: 228 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.048738664Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:20.092211799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.092575779Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.124970754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.125589429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.125631543Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.12565012Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.125667898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717} {"msg":"Crucible 372a1095-279d-48a1-aaa9-14c496bb599f has session id: b768a557-800d-4f09-886d-7529719c256c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.126109807Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Upstairs opts: Upstairs UUID: 372a1095-279d-48a1-aaa9-14c496bb599f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.126144972Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Crucible stats registered with UUID: 372a1095-279d-48a1-aaa9-14c496bb599f","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.126187456Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157583929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717} {"msg":"372a1095-279d-48a1-aaa9-14c496bb599f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157615734Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157633462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157661778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157687875Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157716181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157748996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157780812Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157814917Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157847313Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157881058Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47774)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157913413Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33708)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.157946879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34613)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.1580068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.158737859Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.158774025Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.158794182Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.158835506Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.159037848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.159085822Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.159461739Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.159489106Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.159738481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160029501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160051278Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160065516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160134257Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160151744Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160165193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160302214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160319961Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[0]R flush_numbers[0..12]: [225, 225, 226, 226, 220, 220, 228, 227, 225, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160357536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[0]R generation[0..12]: [129, 129, 130, 130, 124, 124, 133, 132, 129, 124, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160373534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160387582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[1]R flush_numbers[0..12]: [225, 225, 226, 226, 220, 220, 228, 227, 225, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160405849Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[1]R generation[0..12]: [129, 129, 130, 130, 124, 124, 133, 132, 129, 124, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160420147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160434245Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[2]R flush_numbers[0..12]: [225, 225, 226, 226, 220, 220, 228, 227, 225, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160447853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[2]R generation[0..12]: [129, 129, 130, 130, 124, 124, 133, 132, 129, 124, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160461472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160475419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160488228Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160501436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160514294Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160527092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"":"downstairs","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160540351Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160559458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"0","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160572626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"1","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160585714Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"client":"2","":"downstairs","session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160598753Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160613031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"372a1095-279d-48a1-aaa9-14c496bb599f is now active with session: b768a557-800d-4f09-886d-7529719c256c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160625979Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160637977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717,"session_id":"b768a557-800d-4f09-886d-7529719c256c"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:20.160650275Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 265 len 12288 data: 9 8 8 02/30 Write block 97 len 4096 data: 8 03/30 Read block 51 len 24576 04/30 Flush 05/30 Write block 514 len 4096 data: 12 06/30 Read block 120 len 28672 07/30 Read block 530 len 12288 08/30 Write block 350 len 12288 data: 16 18 17 09/30 Read block 541 len 32768 10/30 Flush 11/30 Write block 396 len 20480 data: 8 7 7 6 6 12/30 Read block 537 len 36864 13/30 Read block 337 len 40960 14/30 Read block 168 len 28672 15/30 Write block 396 len 28672 data: 9 8 8 7 7 7 8 16/30 Write block 484 len 4096 data: 12 17/30 Read block 82 len 12288 18/30 Write block 100 len 24576 data: 12 10 9 5 5 5 19/30 Write block 257 len 16384 data: 9 8 7 9 20/30 Write block 79 len 28672 data: 10 10 7 8 10 10 12 21/30 Flush 22/30 Write block 495 len 20480 data: 7 9 10 9 9 23/30 Write block 596 len 16384 data: 4 5 5 3 24/30 Write block 297 len 40960 data: 12 13 13 12 13 12 10 13 14 13 25/30 Read block 192 len 32768 26/30 Write block 60 len 24576 data: 8 10 9 11 12 14 27/30 Write block 475 len 4096 data: 7 28/30 Write block 58 len 36864 data: 10 8 9 11 10 12 13 15 15 29/30 Write block 453 len 16384 data: 9 9 10 12 30/30 Read block 212 len 24576 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Write 4 Done Sent Sent false 30 Acked 1029 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 9 7 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:20.185444901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1717} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 225 225 225 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 130 130 130 226 226 226 T T T 3 060-079 134 134 134 231 231 231 T T T 4 080-099 134 134 134 231 231 231 F F F 5 100-119 134 134 134 231 231 231 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 132 132 132 227 227 227 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 133 133 133 228 228 228 T T T 15 300-319 133 133 133 228 228 228 T T T 16 320-339 126 126 126 221 221 221 F F F 17 340-359 134 134 134 230 230 230 F F F 18 360-379 133 133 133 228 228 228 F F F 19 380-399 134 134 134 231 231 231 F F F 20 400-419 134 134 134 231 231 231 F F F 21 420-439 129 129 129 225 225 225 F F F 22 440-459 130 130 130 226 226 226 T T T 23 460-479 133 133 133 228 228 228 T T T 24 480-499 134 134 134 231 231 231 T T T 25 500-519 134 134 134 230 230 230 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 130 130 130 226 226 226 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 133 133 133 228 228 228 T T T Max gen: 134, Max flush: 231 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 225 225 225 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 135 135 135 232 232 232 F F F 3 060-079 135 135 135 232 232 232 F F F 4 080-099 134 134 134 231 231 231 F F F 5 100-119 134 134 134 231 231 231 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 132 132 132 227 227 227 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 135 135 135 232 232 232 F F F 15 300-319 135 135 135 232 232 232 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 134 134 134 230 230 230 F F F 18 360-379 133 133 133 228 228 228 F F F 19 380-399 134 134 134 231 231 231 F F F 20 400-419 134 134 134 231 231 231 F F F 21 420-439 129 129 129 225 225 225 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 135 135 135 232 232 232 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 134 134 134 230 230 230 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 130 130 130 226 226 226 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 135 135 135 232 232 232 F F F Max gen: 135, Max flush: 232 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.583877288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:24.628133384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.628502383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.661203876Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.661810322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.661853796Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.661871374Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.661888832Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725} {"msg":"Crucible 5404cb00-aa17-44f1-a8e0-cf8b0d0e0dbb has session id: b5d0b0db-c88c-404f-824e-ef43673b3845","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.662371935Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Upstairs opts: Upstairs UUID: 5404cb00-aa17-44f1-a8e0-cf8b0d0e0dbb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.662411279Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Crucible stats registered with UUID: 5404cb00-aa17-44f1-a8e0-cf8b0d0e0dbb","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.662434726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694354597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725} {"msg":"5404cb00-aa17-44f1-a8e0-cf8b0d0e0dbb active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694386043Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.69440331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694430446Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694452923Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694470701Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694487329Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694503986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694530923Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694559539Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694588915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_connection connected from Ok(127.0.0.1:50189)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.69461944Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_connection connected from Ok(127.0.0.1:44754)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694651466Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_connection connected from Ok(127.0.0.1:38123)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.694684371Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.695320083Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.695361498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.695394213Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.695426699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.695533914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.695609454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696153078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696182584Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696219999Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696776042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696802909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696817027Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696829535Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696842973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696856201Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.69686827Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696881618Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[0]R flush_numbers[0..12]: [225, 225, 232, 232, 231, 231, 228, 227, 225, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696895366Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[0]R generation[0..12]: [129, 129, 135, 135, 134, 134, 133, 132, 129, 124, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696909214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696923222Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[1]R flush_numbers[0..12]: [225, 225, 232, 232, 231, 231, 228, 227, 225, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.69693705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[1]R generation[0..12]: [129, 129, 135, 135, 134, 134, 133, 132, 129, 124, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696955148Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696969506Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[2]R flush_numbers[0..12]: [225, 225, 232, 232, 231, 231, 228, 227, 225, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696983084Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[2]R generation[0..12]: [129, 129, 135, 135, 134, 134, 133, 132, 129, 124, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.696997662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.69701164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697024548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697037756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:55:24.697050854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697068412Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"":"downstairs","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.69708196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697093989Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"0","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697107507Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"1","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697120825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"client":"2","":"downstairs","session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697134273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697148771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"5404cb00-aa17-44f1-a8e0-cf8b0d0e0dbb is now active with session: b5d0b0db-c88c-404f-824e-ef43673b3845","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697161669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697173758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725,"session_id":"b5d0b0db-c88c-404f-824e-ef43673b3845"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:24.697185786Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 16384 data: 17 15 13 11 02/30 Read block 481 len 36864 03/30 Write block 62 len 36864 data: 11 13 14 16 16 20 18 20 19 04/30 Read block 220 len 16384 05/30 Write block 376 len 36864 data: 15 18 13 17 15 12 13 14 12 06/30 Write block 12 len 28672 data: 8 9 11 13 13 8 6 07/30 Write block 548 len 24576 data: 8 7 5 5 5 9 08/30 Read block 147 len 12288 09/30 Read block 534 len 16384 10/30 Read block 29 len 12288 11/30 Flush 12/30 Write block 201 len 40960 data: 5 5 7 8 10 11 10 11 12 12 13/30 Read block 396 len 24576 14/30 Write block 422 len 24576 data: 7 5 7 8 8 8 15/30 Read block 470 len 12288 16/30 Read block 388 len 4096 17/30 Write block 518 len 8192 data: 11 12 18/30 Write block 507 len 40960 data: 7 5 6 5 4 8 9 13 14 13 19/30 Read block 91 len 8192 20/30 Read block 491 len 8192 21/30 Write block 312 len 28672 data: 8 7 6 6 7 9 8 22/30 Write block 502 len 8192 data: 11 10 23/30 Write block 486 len 12288 data: 10 8 7 24/30 Read block 505 len 32768 25/30 Read block 56 len 36864 26/30 Read block 178 len 12288 27/30 Write block 296 len 20480 data: 12 13 14 14 13 28/30 Write block 463 len 8192 data: 14 14 29/30 Write block 415 len 16384 data: 9 9 10 13 30/30 Write block 426 len 40960 data: 9 9 6 6 6 7 8 5 5 5 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 12 Acked 1011 Write 10 Done Done Done false 13 Acked 1012 Read 6 Done Done Done false 14 Acked 1013 Write 6 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 1 Done Done Done false 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Write 10 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:24.719762167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1725} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 135 135 135 232 232 232 F F F 3 060-079 136 136 136 233 233 233 F F F 4 080-099 134 134 134 231 231 231 F F F 5 100-119 134 134 134 231 231 231 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 132 132 132 227 227 227 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 133 133 133 228 228 228 T T T 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 135 135 135 232 232 232 T T T 15 300-319 135 135 135 232 232 232 T T T 16 320-339 126 126 126 221 221 221 F F F 17 340-359 134 134 134 230 230 230 F F F 18 360-379 136 136 136 233 233 233 F F F 19 380-399 136 136 136 233 233 233 F F F 20 400-419 136 136 136 233 233 233 T T T 21 420-439 129 129 129 225 225 225 T T T 22 440-459 135 135 135 232 232 232 F F F 23 460-479 135 135 135 232 232 232 T T T 24 480-499 135 135 135 232 232 232 T T T 25 500-519 134 134 134 230 230 230 T T T 26 520-539 133 133 133 228 228 228 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 135 135 135 232 232 232 F F F Max gen: 136, Max flush: 233 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 135 135 135 232 232 232 F F F 3 060-079 136 136 136 233 233 233 F F F 4 080-099 134 134 134 231 231 231 F F F 5 100-119 134 134 134 231 231 231 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 132 132 132 227 227 227 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 F F F 10 200-219 137 137 137 234 234 234 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 137 137 137 234 234 234 F F F 15 300-319 137 137 137 234 234 234 F F F 16 320-339 126 126 126 221 221 221 F F F 17 340-359 134 134 134 230 230 230 F F F 18 360-379 136 136 136 233 233 233 F F F 19 380-399 136 136 136 233 233 233 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 137 137 137 234 234 234 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 137 137 137 234 234 234 F F F 24 480-499 137 137 137 234 234 234 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 135 135 135 232 232 232 F F F Max gen: 137, Max flush: 234 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.10229928Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:29.147269218Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.14790222Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.181851421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.182874099Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.182910774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.182925463Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.182939841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733} {"msg":"Crucible a533a9e3-a354-46be-b62a-4e6b11124e26 has session id: 0f778112-54cb-413e-b998-83f9f811cc6c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.183475976Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Upstairs opts: Upstairs UUID: a533a9e3-a354-46be-b62a-4e6b11124e26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.183527829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Crucible stats registered with UUID: a533a9e3-a354-46be-b62a-4e6b11124e26","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.183549486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216471349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733} {"msg":"a533a9e3-a354-46be-b62a-4e6b11124e26 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216542999Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216590662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216644645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216668501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216686739Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216704117Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216762988Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216794194Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216828219Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216856356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60862)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.216918327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52666)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.217372934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40753)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.217425537Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.217852118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.217879344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.218076437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.218110702Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.218132589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.218146777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.218553851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.218595825Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.218892344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219136541Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219166016Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219182054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219272052Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219291429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219305527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219530986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219550113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[0]R flush_numbers[0..12]: [233, 225, 232, 233, 231, 231, 228, 227, 225, 220, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.21957709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[0]R generation[0..12]: [136, 129, 135, 136, 134, 134, 133, 132, 129, 124, 137, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219592458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219611075Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[1]R flush_numbers[0..12]: [233, 225, 232, 233, 231, 231, 228, 227, 225, 220, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219625683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[1]R generation[0..12]: [136, 129, 135, 136, 134, 134, 133, 132, 129, 124, 137, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219639571Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219653629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[2]R flush_numbers[0..12]: [233, 225, 232, 233, 231, 231, 228, 227, 225, 220, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219667107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[2]R generation[0..12]: [136, 129, 135, 136, 134, 134, 133, 132, 129, 124, 137, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219680885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219694683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219707771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.21972084Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219733738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219746626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"":"downstairs","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219760034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219772073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"0","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219797029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"1","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219827155Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"client":"2","":"downstairs","session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219855321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219880138Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"a533a9e3-a354-46be-b62a-4e6b11124e26 is now active with session: 0f778112-54cb-413e-b998-83f9f811cc6c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219906264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219930991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733,"session_id":"0f778112-54cb-413e-b998-83f9f811cc6c"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:29.219956727Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 55 len 24576 data: 10 12 13 11 9 10 03/30 Write block 158 len 4096 data: 11 04/30 Read block 542 len 24576 05/30 Read block 233 len 28672 06/30 Write block 489 len 12288 data: 9 10 9 07/30 Write block 297 len 16384 data: 14 15 15 14 08/30 Read block 223 len 40960 09/30 Write block 379 len 20480 data: 18 16 13 14 15 10/30 Write block 60 len 32768 data: 11 12 12 14 15 17 17 21 11/30 Read block 500 len 40960 12/30 Flush 13/30 Write block 370 len 12288 data: 10 9 11 14/30 Write block 76 len 24576 data: 13 14 12 11 11 8 15/30 Read block 447 len 12288 16/30 Read block 410 len 36864 17/30 Write block 438 len 8192 data: 6 5 18/30 Read block 202 len 20480 19/30 Write block 314 len 24576 data: 7 7 8 10 9 11 20/30 Write block 146 len 20480 data: 12 11 10 10 8 21/30 Read block 176 len 20480 22/30 Read block 545 len 24576 23/30 Write block 339 len 28672 data: 6 6 9 10 7 9 8 24/30 Write block 189 len 40960 data: 7 11 10 10 10 12 11 11 8 7 25/30 Write block 590 len 24576 data: 7 8 8 6 6 6 26/30 Write block 568 len 8192 data: 11 11 27/30 Read block 177 len 4096 28/30 Read block 161 len 24576 29/30 Read block 541 len 12288 30/30 Write block 64 len 16384 data: 16 18 18 22 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 3 Done Done Done false 14 Acked 1013 Write 6 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 9 Done Done Done false 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:29.242055624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1733} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 138 138 138 236 236 236 F F F 3 060-079 138 138 138 236 236 236 T T T 4 080-099 134 134 134 231 231 231 T T T 5 100-119 134 134 134 231 231 231 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 138 138 138 236 236 236 T T T 8 160-179 129 129 129 225 225 225 F F F 9 180-199 124 124 124 220 220 220 T T T 10 200-219 137 137 137 234 234 234 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 138 138 138 236 236 236 F F F 15 300-319 138 138 138 236 236 236 T T T 16 320-339 126 126 126 221 221 221 T T T 17 340-359 134 134 134 230 230 230 T T T 18 360-379 138 138 138 236 236 236 T T T 19 380-399 138 138 138 236 236 236 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 137 137 137 234 234 234 T T T 22 440-459 135 135 135 232 232 232 F F F 23 460-479 137 137 137 234 234 234 F F F 24 480-499 138 138 138 236 236 236 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 133 133 133 228 228 228 T T T 29 580-599 135 135 135 232 232 232 T T T Max gen: 138, Max flush: 236 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 129 129 129 225 225 225 F F F 2 040-059 138 138 138 236 236 236 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 139 139 139 237 237 237 F F F 5 100-119 134 134 134 231 231 231 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 139 139 139 237 237 237 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 139 139 139 237 237 237 F F F 10 200-219 137 137 137 234 234 234 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 138 138 138 236 236 236 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 139 139 139 237 237 237 F F F 18 360-379 139 139 139 237 237 237 F F F 19 380-399 138 138 138 236 236 236 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 139 139 139 237 237 237 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 137 137 137 234 234 234 F F F 24 480-499 138 138 138 236 236 236 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 139 139 139 237 237 237 F F F 29 580-599 139 139 139 237 237 237 F F F Max gen: 139, Max flush: 237 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.691157033Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.737262864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.738192966Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.7382295Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.738244818Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.738259206Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741} {"msg":"Crucible 50c87e9e-db1c-4f2d-aad5-cfebda1108af has session id: 9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.738752778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Upstairs opts: Upstairs UUID: 50c87e9e-db1c-4f2d-aad5-cfebda1108af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.738785054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Crucible stats registered with UUID: 50c87e9e-db1c-4f2d-aad5-cfebda1108af","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.738802221Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.770867432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741} {"msg":"50c87e9e-db1c-4f2d-aad5-cfebda1108af active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.770922914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.770965628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771020651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771059826Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771094511Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771121027Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771139415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771156442Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.77117419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771191437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64915)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771221913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61034)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771256178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55249)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.771288764Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.772169462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.772196569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.772211066Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.772226694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.772363445Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.77240202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.77290703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.772948394Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773009176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773561619Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773589156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773604174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773616942Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.77363074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773644178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773656276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773669465Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[0]R flush_numbers[0..12]: [233, 225, 236, 237, 237, 231, 228, 237, 225, 237, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773687692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[0]R generation[0..12]: [136, 129, 138, 139, 139, 134, 133, 139, 129, 139, 137, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.77370159Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773720447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[1]R flush_numbers[0..12]: [233, 225, 236, 237, 237, 231, 228, 237, 225, 237, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773734705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[1]R generation[0..12]: [136, 129, 138, 139, 139, 134, 133, 139, 129, 139, 137, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773748523Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773762452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[2]R flush_numbers[0..12]: [233, 225, 236, 237, 237, 231, 228, 237, 225, 237, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.77377615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[2]R generation[0..12]: [136, 129, 138, 139, 139, 134, 133, 139, 129, 139, 137, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773790468Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773804236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773817054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773830092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773842781Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773855569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"":"downstairs","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773868967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773881005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"0","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773894423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"1","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773910001Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"client":"2","":"downstairs","session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773923649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"Wait for a query_work_queue command to finish before sending IO 9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773946766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"50c87e9e-db1c-4f2d-aad5-cfebda1108af is now active with session: 9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773959634Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773971493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741,"session_id":"9dfb10d6-e1be-4d3e-bc14-a2f9cfcdd13f"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:33.773983671Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 38 len 8192 data: 11 12 02/30 Flush 03/30 Write block 205 len 4096 data: 11 04/30 Read block 222 len 36864 05/30 Write block 92 len 8192 data: 8 7 06/30 Read block 284 len 12288 07/30 Write block 479 len 8192 data: 13 14 08/30 Read block 11 len 16384 09/30 Read block 487 len 8192 10/30 Write block 395 len 12288 data: 9 10 9 11/30 Read block 192 len 36864 12/30 Read block 491 len 20480 13/30 Write block 426 len 40960 data: 10 10 7 7 7 8 9 6 6 6 14/30 Write block 432 len 36864 data: 10 7 7 7 6 6 7 6 5 15/30 Read block 532 len 12288 16/30 Read block 102 len 8192 17/30 Flush 18/30 Write block 106 len 20480 data: 5 7 8 7 8 19/30 Read block 320 len 36864 20/30 Read block 389 len 32768 21/30 Read block 188 len 16384 22/30 Write block 390 len 12288 data: 14 11 10 23/30 Write block 399 len 4096 data: 8 24/30 Flush 25/30 Write block 375 len 12288 data: 11 16 19 26/30 Read block 529 len 36864 27/30 Write block 365 len 12288 data: 11 11 13 28/30 Read block 388 len 36864 29/30 Write block 346 len 20480 data: 12 13 15 18 17 30/30 Read block 273 len 20480 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 5 Done Sent Sent false 30 Acked 1029 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 6 4 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:33.798650304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1741} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 140 140 140 238 238 238 F F F 2 040-059 138 138 138 236 236 236 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 140 140 140 239 239 239 F F F 5 100-119 140 140 140 240 240 240 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 139 139 139 237 237 237 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 139 139 139 237 237 237 F F F 10 200-219 140 140 140 239 239 239 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 138 138 138 236 236 236 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 139 139 139 237 237 237 T T T 18 360-379 139 139 139 237 237 237 T T T 19 380-399 140 140 140 240 240 240 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 239 239 239 F F F 22 440-459 140 140 140 239 239 239 F F F 23 460-479 140 140 140 239 239 239 F F F 24 480-499 140 140 140 239 239 239 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 139 139 139 237 237 237 F F F 29 580-599 139 139 139 237 237 237 F F F Max gen: 140, Max flush: 240 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 140 140 140 238 238 238 F F F 2 040-059 138 138 138 236 236 236 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 140 140 140 239 239 239 F F F 5 100-119 140 140 140 240 240 240 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 139 139 139 237 237 237 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 139 139 139 237 237 237 F F F 10 200-219 140 140 140 239 239 239 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 138 138 138 236 236 236 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 141 141 141 241 241 241 F F F 18 360-379 141 141 141 241 241 241 F F F 19 380-399 140 140 140 240 240 240 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 239 239 239 F F F 22 440-459 140 140 140 239 239 239 F F F 23 460-479 140 140 140 239 239 239 F F F 24 480-499 140 140 140 239 239 239 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 139 139 139 237 237 237 F F F 29 580-599 139 139 139 237 237 237 F F F Max gen: 141, Max flush: 241 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.075074404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:38.119308844Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.119734195Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.152795438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.153468725Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.153513259Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.153530667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.153548444Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749} {"msg":"Crucible f696ffd0-b82c-4eea-b516-8ac5fede7ed6 has session id: 233885d3-f802-4dec-8189-3e7579eb06f2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.15408864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Upstairs opts: Upstairs UUID: f696ffd0-b82c-4eea-b516-8ac5fede7ed6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.154130244Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Crucible stats registered with UUID: f696ffd0-b82c-4eea-b516-8ac5fede7ed6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.154150911Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.18629756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749} {"msg":"f696ffd0-b82c-4eea-b516-8ac5fede7ed6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186346924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186378229Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186425093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186469207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186503542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186541667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186581371Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186616347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186653091Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186689876Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61358)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186730801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:57145)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.186770805Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:42342)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.18680565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.187552957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.187591092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.187615158Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.187638645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.187811711Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.187855885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.188306383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.188334869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.188350387Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.188883473Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.18890805Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.188922138Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189014815Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189036422Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.18904996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189062269Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189075547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[0]R flush_numbers[0..12]: [233, 238, 236, 237, 239, 240, 228, 237, 225, 237, 239, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189089445Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[0]R generation[0..12]: [136, 140, 138, 139, 140, 140, 133, 139, 129, 139, 140, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189103573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189117811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[1]R flush_numbers[0..12]: [233, 238, 236, 237, 239, 240, 228, 237, 225, 237, 239, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189136278Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[1]R generation[0..12]: [136, 140, 138, 139, 140, 140, 133, 139, 129, 139, 140, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189150626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189164515Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[2]R flush_numbers[0..12]: [233, 238, 236, 237, 239, 240, 228, 237, 225, 237, 239, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189179082Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[2]R generation[0..12]: [136, 140, 138, 139, 140, 140, 133, 139, 129, 139, 140, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.18919526Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189211598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189226756Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189242324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189257712Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189273089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"":"downstairs","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189288997Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189303385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"0","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Transition from WaitQuorum to Active","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189327782Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"1","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.18934617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"client":"2","":"downstairs","session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189374386Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189405201Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"f696ffd0-b82c-4eea-b516-8ac5fede7ed6 is now active with session: 233885d3-f802-4dec-8189-3e7579eb06f2","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189435617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189460014Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749,"session_id":"233885d3-f802-4dec-8189-3e7579eb06f2"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:38.189473622Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 16384 data: 9 7 3 4 02/30 Read block 362 len 32768 03/30 Write block 202 len 8192 data: 6 8 04/30 Write block 527 len 8192 data: 10 12 05/30 Read block 310 len 36864 06/30 Read block 199 len 28672 07/30 Write block 568 len 20480 data: 12 12 10 10 10 08/30 Read block 356 len 16384 09/30 Read block 64 len 20480 10/30 Read block 291 len 32768 11/30 Read block 57 len 24576 12/30 Read block 459 len 32768 13/30 Flush 14/30 Flush 15/30 Write block 117 len 40960 data: 13 13 15 14 13 13 10 11 13 14 16/30 Read block 374 len 4096 17/30 Flush 18/30 Write block 547 len 12288 data: 9 9 8 19/30 Write block 277 len 28672 data: 12 12 12 11 11 9 10 20/30 Read block 521 len 8192 21/30 Read block 88 len 4096 22/30 Read block 91 len 12288 23/30 Read block 418 len 24576 24/30 Read block 41 len 24576 25/30 Write block 446 len 20480 data: 4 6 9 10 8 26/30 Flush 27/30 Write block 130 len 20480 data: 16 16 16 14 13 28/30 Read block 221 len 20480 29/30 Write block 116 len 40960 data: 13 14 14 16 15 14 14 11 12 14 30/30 Read block 308 len 12288 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 10 Done Sent Sent false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:38.241698208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1749} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 142 142 142 242 242 242 F F F 2 040-059 138 138 138 236 236 236 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 140 140 140 239 239 239 F F F 5 100-119 142 142 142 244 244 244 T T T 6 120-139 142 142 142 244 244 244 T T T 7 140-159 139 139 139 237 237 237 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 139 139 139 237 237 237 F F F 10 200-219 142 142 142 242 242 242 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 142 142 142 245 245 245 F F F 14 280-299 142 142 142 245 245 245 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 141 141 141 241 241 241 F F F 18 360-379 141 141 141 241 241 241 F F F 19 380-399 140 140 140 240 240 240 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 239 239 239 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 140 140 140 239 239 239 F F F 24 480-499 140 140 140 239 239 239 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 142 142 142 245 245 245 F F F 28 560-579 142 142 142 242 242 242 F F F 29 580-599 139 139 139 237 237 237 F F F Max gen: 142, Max flush: 245 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 142 142 142 242 242 242 F F F 2 040-059 138 138 138 236 236 236 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 140 140 140 239 239 239 F F F 5 100-119 143 143 143 246 246 246 F F F 6 120-139 143 143 143 246 246 246 F F F 7 140-159 139 139 139 237 237 237 F F F 8 160-179 129 129 129 225 225 225 F F F 9 180-199 139 139 139 237 237 237 F F F 10 200-219 142 142 142 242 242 242 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 142 142 142 245 245 245 F F F 14 280-299 142 142 142 245 245 245 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 141 141 141 241 241 241 F F F 18 360-379 141 141 141 241 241 241 F F F 19 380-399 140 140 140 240 240 240 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 239 239 239 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 140 140 140 239 239 239 F F F 24 480-499 140 140 140 239 239 239 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 142 142 142 245 245 245 F F F 28 560-579 142 142 142 242 242 242 F F F 29 580-599 139 139 139 237 237 237 F F F Max gen: 143, Max flush: 246 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.553530477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.600073288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.601082819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.601118304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.601138301Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.601153169Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757} {"msg":"Crucible 9f88d4af-d160-47e0-8734-0a6e0f8d2f2c has session id: 77408d9e-6e24-4e02-ae69-747cf3da6bf9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.601633893Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Upstairs opts: Upstairs UUID: 9f88d4af-d160-47e0-8734-0a6e0f8d2f2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.601673447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Crucible stats registered with UUID: 9f88d4af-d160-47e0-8734-0a6e0f8d2f2c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.601697364Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.633750346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757} {"msg":"9f88d4af-d160-47e0-8734-0a6e0f8d2f2c active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.633813258Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.633855542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.633902565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.633934611Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.633965587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.633984504Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634001602Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634024698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634042706Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634060124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51694)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634077491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41175)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634094749Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38501)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634320488Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634940162Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634968248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.634999564Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.635014022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.635143154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.635181589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.635621918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.635781986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.635815921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636201288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636272938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636302984Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636318202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636333379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636346918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636428806Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636448584Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[0]R flush_numbers[0..12]: [233, 242, 236, 237, 239, 246, 246, 237, 225, 237, 242, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636470181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[0]R generation[0..12]: [136, 142, 138, 139, 140, 143, 143, 139, 129, 139, 142, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636484379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636498617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[1]R flush_numbers[0..12]: [233, 242, 236, 237, 239, 246, 246, 237, 225, 237, 242, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636512305Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[1]R generation[0..12]: [136, 142, 138, 139, 140, 143, 143, 139, 129, 139, 142, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636525973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636539641Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[2]R flush_numbers[0..12]: [233, 242, 236, 237, 239, 246, 246, 237, 225, 237, 242, 228]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636553289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[2]R generation[0..12]: [136, 142, 138, 139, 140, 143, 143, 139, 129, 139, 142, 133]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636570707Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636593794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.63661963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636643837Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636667253Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636699669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"":"downstairs","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636725955Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636741303Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"0","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636754661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"1","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.63676769Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"client":"2","":"downstairs","session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636790926Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636814303Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"9f88d4af-d160-47e0-8734-0a6e0f8d2f2c is now active with session: 77408d9e-6e24-4e02-ae69-747cf3da6bf9","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.63683855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636852498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757,"session_id":"77408d9e-6e24-4e02-ae69-747cf3da6bf9"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:42.636873345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 265 len 24576 data: 10 9 9 11 9 9 02/30 Write block 233 len 16384 data: 8 10 8 8 03/30 Flush 04/30 Write block 195 len 16384 data: 12 12 9 8 05/30 Flush 06/30 Write block 348 len 4096 data: 16 07/30 Flush 08/30 Write block 346 len 28672 data: 13 14 17 19 18 19 18 09/30 Write block 165 len 8192 data: 7 6 10/30 Read block 523 len 20480 11/30 Read block 346 len 20480 12/30 Read block 525 len 40960 13/30 Read block 348 len 40960 14/30 Write block 471 len 36864 data: 11 11 8 9 8 9 9 9 14 15/30 Flush 16/30 Write block 486 len 12288 data: 11 9 8 17/30 Read block 286 len 32768 18/30 Read block 226 len 28672 19/30 Read block 425 len 28672 20/30 Write block 28 len 32768 data: 7 8 9 9 8 10 10 7 21/30 Write block 135 len 28672 data: 9 11 10 12 14 12 12 22/30 Write block 193 len 32768 data: 11 13 13 13 10 9 6 5 23/30 Read block 265 len 32768 24/30 Write block 340 len 8192 data: 7 10 25/30 Read block 84 len 28672 26/30 Write block 550 len 32768 data: 6 6 6 10 11 11 12 9 27/30 Write block 16 len 8192 data: 14 9 28/30 Write block 474 len 40960 data: 10 9 10 10 10 15 15 12 13 13 29/30 Read block 465 len 32768 30/30 Read block 373 len 32768 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 3 Done Done Done false 17 Acked 1016 Read 8 Done Done Done false 18 Acked 1017 Read 7 Done Done Done false 19 Acked 1018 Read 7 Done Done Done false 20 Acked 1019 Write 8 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 14 15 15 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:42.664248723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1757} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 T T T 1 020-039 142 142 142 242 242 242 T T T 2 040-059 138 138 138 236 236 236 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 140 140 140 239 239 239 F F F 5 100-119 143 143 143 246 246 246 F F F 6 120-139 143 143 143 246 246 246 T T T 7 140-159 139 139 139 237 237 237 T T T 8 160-179 144 144 144 250 250 250 F F F 9 180-199 144 144 144 248 248 248 T T T 10 200-219 142 142 142 242 242 242 T T T 11 220-239 144 144 144 247 247 247 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 144 144 144 247 247 247 F F F 14 280-299 142 142 142 245 245 245 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 144 144 144 250 250 250 T T T 18 360-379 141 141 141 241 241 241 F F F 19 380-399 140 140 140 240 240 240 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 239 239 239 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 144 144 144 250 250 250 T T T 24 480-499 140 140 140 239 239 239 T T T 25 500-519 137 137 137 234 234 234 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 142 142 142 245 245 245 T T T 28 560-579 142 142 142 242 242 242 F F F 29 580-599 139 139 139 237 237 237 F F F Max gen: 144, Max flush: 250 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 251 251 251 F F F 1 020-039 145 145 145 251 251 251 F F F 2 040-059 138 138 138 236 236 236 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 140 140 140 239 239 239 F F F 5 100-119 143 143 143 246 246 246 F F F 6 120-139 145 145 145 251 251 251 F F F 7 140-159 145 145 145 251 251 251 F F F 8 160-179 144 144 144 250 250 250 F F F 9 180-199 145 145 145 251 251 251 F F F 10 200-219 145 145 145 251 251 251 F F F 11 220-239 144 144 144 247 247 247 F F F 12 240-259 134 134 134 231 231 231 F F F 13 260-279 144 144 144 247 247 247 F F F 14 280-299 142 142 142 245 245 245 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 145 145 145 251 251 251 F F F 18 360-379 141 141 141 241 241 241 F F F 19 380-399 140 140 140 240 240 240 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 239 239 239 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 145 145 145 251 251 251 F F F 28 560-579 142 142 142 242 242 242 F F F 29 580-599 139 139 139 237 237 237 F F F Max gen: 145, Max flush: 251 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.042733685Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:47.086024456Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.086431369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.118299627Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.11893245Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.118969954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.118986142Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.11900365Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765} {"msg":"Crucible 005c644c-3a94-4a62-8030-fd1127cd45d0 has session id: fb548445-70b3-44ae-bbad-634a421b2181","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.119471225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Upstairs opts: Upstairs UUID: 005c644c-3a94-4a62-8030-fd1127cd45d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.119517909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Crucible stats registered with UUID: 005c644c-3a94-4a62-8030-fd1127cd45d0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.119541795Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150788079Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765} {"msg":"005c644c-3a94-4a62-8030-fd1127cd45d0 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150814556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150831084Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.15085936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150876797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150893895Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150910693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150932039Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150949037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.150969554Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.15100137Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_connection connected from Ok(127.0.0.1:38073)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.15114704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_connection connected from Ok(127.0.0.1:54661)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.151189884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_connection connected from Ok(127.0.0.1:60578)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.151260774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152007411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152034837Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152050105Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152063823Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152093209Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152118906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152685307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152819889Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.152852494Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153263197Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153290264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153305652Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.15339415Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153421855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153436724Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153449412Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.15346298Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[0]R flush_numbers[0..12]: [251, 251, 236, 237, 239, 246, 251, 251, 250, 251, 251, 247]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153476818Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[0]R generation[0..12]: [145, 145, 138, 139, 140, 143, 145, 145, 144, 145, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153490766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153504614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[1]R flush_numbers[0..12]: [251, 251, 236, 237, 239, 246, 251, 251, 250, 251, 251, 247]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153518462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[1]R generation[0..12]: [145, 145, 138, 139, 140, 143, 145, 145, 144, 145, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.15353236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153546668Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[2]R flush_numbers[0..12]: [251, 251, 236, 237, 239, 246, 251, 251, 250, 251, 251, 247]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153564496Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[2]R generation[0..12]: [145, 145, 138, 139, 140, 143, 145, 145, 144, 145, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153578864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153593072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.1536062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153619588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153632397Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153649934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"":"downstairs","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153670351Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153703007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"0","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153730553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"1","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.15374811Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"client":"2","":"downstairs","session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153761909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153781546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"005c644c-3a94-4a62-8030-fd1127cd45d0 is now active with session: fb548445-70b3-44ae-bbad-634a421b2181","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153805662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153843127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765,"session_id":"fb548445-70b3-44ae-bbad-634a421b2181"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:47.153866744Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 12288 data: 9 8 9 02/30 Read block 165 len 32768 03/30 Write block 86 len 12288 data: 11 11 13 04/30 Read block 454 len 40960 05/30 Read block 52 len 4096 06/30 Write block 425 len 32768 data: 9 11 11 8 8 8 9 11 07/30 Write block 257 len 20480 data: 10 9 8 10 12 08/30 Write block 563 len 16384 data: 9 10 11 12 09/30 Write block 98 len 16384 data: 10 12 13 11 10/30 Write block 398 len 28672 data: 9 9 8 8 9 10 11 11/30 Write block 402 len 4096 data: 10 12/30 Flush 13/30 Write block 584 len 32768 data: 9 10 10 7 8 6 8 9 14/30 Read block 549 len 36864 15/30 Flush 16/30 Write block 106 len 8192 data: 6 8 17/30 Write block 49 len 16384 data: 17 14 15 12 18/30 Write block 242 len 36864 data: 7 10 12 13 19 21 20 19 15 19/30 Read block 166 len 4096 20/30 Flush 21/30 Write block 289 len 40960 data: 8 8 8 9 11 9 9 13 15 16 22/30 Flush 23/30 Write block 551 len 36864 data: 7 7 11 12 12 13 10 13 12 24/30 Write block 589 len 8192 data: 7 9 25/30 Read block 323 len 4096 26/30 Read block 594 len 8192 27/30 Read block 373 len 4096 28/30 Write block 238 len 40960 data: 8 9 9 7 8 11 13 14 20 22 29/30 Write block 9 len 40960 data: 12 14 9 9 10 12 14 15 10 7 30/30 Read block 372 len 32768 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 7 7 8 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:47.182479352Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1765} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 251 251 251 T T T 1 020-039 145 145 145 251 251 251 F F F 2 040-059 146 146 146 254 254 254 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 146 146 146 252 252 252 F F F 5 100-119 146 146 146 254 254 254 F F F 6 120-139 145 145 145 251 251 251 F F F 7 140-159 145 145 145 251 251 251 F F F 8 160-179 144 144 144 250 250 250 F F F 9 180-199 145 145 145 251 251 251 F F F 10 200-219 145 145 145 251 251 251 F F F 11 220-239 144 144 144 247 247 247 T T T 12 240-259 146 146 146 254 254 254 T T T 13 260-279 146 146 146 252 252 252 F F F 14 280-299 146 146 146 255 255 255 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 145 145 145 251 251 251 F F F 18 360-379 141 141 141 241 241 241 F F F 19 380-399 146 146 146 252 252 252 F F F 20 400-419 146 146 146 252 252 252 F F F 21 420-439 146 146 146 252 252 252 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 145 145 145 251 251 251 T T T 28 560-579 146 146 146 252 252 252 F F F 29 580-599 146 146 146 253 253 253 T T T Max gen: 146, Max flush: 255 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 256 256 256 F F F 1 020-039 145 145 145 251 251 251 F F F 2 040-059 146 146 146 254 254 254 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 146 146 146 252 252 252 F F F 5 100-119 146 146 146 254 254 254 F F F 6 120-139 145 145 145 251 251 251 F F F 7 140-159 145 145 145 251 251 251 F F F 8 160-179 144 144 144 250 250 250 F F F 9 180-199 145 145 145 251 251 251 F F F 10 200-219 145 145 145 251 251 251 F F F 11 220-239 147 147 147 256 256 256 F F F 12 240-259 147 147 147 256 256 256 F F F 13 260-279 146 146 146 252 252 252 F F F 14 280-299 146 146 146 255 255 255 F F F 15 300-319 139 139 139 237 237 237 F F F 16 320-339 139 139 139 237 237 237 F F F 17 340-359 145 145 145 251 251 251 F F F 18 360-379 141 141 141 241 241 241 F F F 19 380-399 146 146 146 252 252 252 F F F 20 400-419 146 146 146 252 252 252 F F F 21 420-439 146 146 146 252 252 252 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 147 147 147 256 256 256 F F F 28 560-579 146 146 146 252 252 252 F F F 29 580-599 147 147 147 256 256 256 F F F Max gen: 147, Max flush: 256 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.532478709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.57699742Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.577640821Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.577676366Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.577696223Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.577711491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773} {"msg":"Crucible 916a45ae-5fe1-4abc-b7b9-bedbc55dd9c6 has session id: 9b8102e1-2548-425e-abcb-c854965c516a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.578159729Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Upstairs opts: Upstairs UUID: 916a45ae-5fe1-4abc-b7b9-bedbc55dd9c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.578198384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Crucible stats registered with UUID: 916a45ae-5fe1-4abc-b7b9-bedbc55dd9c6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.5782234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610640672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773} {"msg":"916a45ae-5fe1-4abc-b7b9-bedbc55dd9c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610686076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610705033Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610734289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610751977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610768675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610784953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.61080154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610818218Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610839125Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.61087588Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34617)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.610907436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43660)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.61123646Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55154)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.611281664Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.611783395Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.611820399Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.611846276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.611931184Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.61196018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.611986847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.612472809Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.612520493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.612815322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613067607Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613132618Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613156425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613182841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613198129Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613211737Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613441675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613461893Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[0]R flush_numbers[0..12]: [256, 251, 254, 237, 252, 254, 251, 251, 250, 251, 251, 256]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.61348353Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[0]R generation[0..12]: [147, 145, 146, 139, 146, 146, 145, 145, 144, 145, 145, 147]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613497778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613511816Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[1]R flush_numbers[0..12]: [256, 251, 254, 237, 252, 254, 251, 251, 250, 251, 251, 256]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613525854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[1]R generation[0..12]: [147, 145, 146, 139, 146, 146, 145, 145, 144, 145, 145, 147]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613539702Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.6135537Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[2]R flush_numbers[0..12]: [256, 251, 254, 237, 252, 254, 251, 251, 250, 251, 251, 256]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613567578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[2]R generation[0..12]: [147, 145, 146, 139, 146, 146, 145, 145, 144, 145, 145, 147]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613581216Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613597994Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613611472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.61362484Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613638198Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613651306Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"":"downstairs","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613664455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613676533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"0","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613689901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"1","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:55:51.613702829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"client":"2","":"downstairs","session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613719387Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613731726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"916a45ae-5fe1-4abc-b7b9-bedbc55dd9c6 is now active with session: 9b8102e1-2548-425e-abcb-c854965c516a","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613744614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.613756782Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773,"session_id":"9b8102e1-2548-425e-abcb-c854965c516a"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:51.61376905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 355 len 36864 data: 12 11 12 10 10 14 15 14 14 02/30 Read block 556 len 28672 03/30 Flush 04/30 Write block 224 len 8192 data: 12 12 05/30 Write block 374 len 8192 data: 9 12 06/30 Read block 501 len 36864 07/30 Write block 298 len 16384 data: 17 16 15 14 08/30 Read block 429 len 28672 09/30 Read block 7 len 20480 10/30 Write block 417 len 24576 data: 11 14 12 10 9 8 11/30 Read block 125 len 4096 12/30 Read block 44 len 28672 13/30 Read block 529 len 12288 14/30 Read block 192 len 40960 15/30 Flush 16/30 Write block 417 len 8192 data: 12 15 17/30 Write block 105 len 32768 data: 6 7 9 10 9 10 7 7 18/30 Flush 19/30 Write block 510 len 8192 data: 6 5 20/30 Read block 482 len 20480 21/30 Read block 8 len 16384 22/30 Read block 55 len 32768 23/30 Read block 352 len 36864 24/30 Read block 391 len 20480 25/30 Write block 53 len 16384 data: 11 10 11 13 26/30 Write block 233 len 32768 data: 9 11 9 9 7 9 10 10 27/30 Write block 89 len 32768 data: 12 11 10 9 8 6 8 8 28/30 Write block 592 len 16384 data: 9 7 7 7 29/30 Write block 321 len 20480 data: 11 13 13 14 13 30/30 Write block 566 len 32768 data: 13 12 13 13 11 11 11 11 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 5 Done Done Sent false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:51.639141785Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1773} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 256 256 256 F F F 1 020-039 145 145 145 251 251 251 F F F 2 040-059 146 146 146 254 254 254 T T T 3 060-079 139 139 139 237 237 237 F F F 4 080-099 146 146 146 252 252 252 T T T 5 100-119 148 148 148 259 259 259 F F F 6 120-139 145 145 145 251 251 251 F F F 7 140-159 145 145 145 251 251 251 F F F 8 160-179 144 144 144 250 250 250 F F F 9 180-199 145 145 145 251 251 251 F F F 10 200-219 145 145 145 251 251 251 F F F 11 220-239 148 148 148 258 258 258 T T T 12 240-259 147 147 147 256 256 256 T T T 13 260-279 146 146 146 252 252 252 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 148 148 148 258 258 258 F F F 16 320-339 139 139 139 237 237 237 T T T 17 340-359 148 148 148 257 257 257 F F F 18 360-379 148 148 148 258 258 258 F F F 19 380-399 146 146 146 252 252 252 F F F 20 400-419 148 148 148 259 259 259 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 137 137 137 234 234 234 T T T 26 520-539 142 142 142 242 242 242 F F F 27 540-559 147 147 147 256 256 256 F F F 28 560-579 146 146 146 252 252 252 T T T 29 580-599 147 147 147 256 256 256 T T T Max gen: 148, Max flush: 259 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 256 256 256 F F F 1 020-039 145 145 145 251 251 251 F F F 2 040-059 149 149 149 260 260 260 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 148 148 148 259 259 259 F F F 6 120-139 145 145 145 251 251 251 F F F 7 140-159 145 145 145 251 251 251 F F F 8 160-179 144 144 144 250 250 250 F F F 9 180-199 145 145 145 251 251 251 F F F 10 200-219 145 145 145 251 251 251 F F F 11 220-239 149 149 149 260 260 260 F F F 12 240-259 149 149 149 260 260 260 F F F 13 260-279 146 146 146 252 252 252 F F F 14 280-299 148 148 148 258 258 258 F F F 15 300-319 148 148 148 258 258 258 F F F 16 320-339 149 149 149 260 260 260 F F F 17 340-359 148 148 148 257 257 257 F F F 18 360-379 148 148 148 258 258 258 F F F 19 380-399 146 146 146 252 252 252 F F F 20 400-419 148 148 148 259 259 259 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 149 149 149 260 260 260 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 147 147 147 256 256 256 F F F 28 560-579 149 149 149 260 260 260 F F F 29 580-599 149 149 149 260 260 260 F F F Max gen: 149, Max flush: 260 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:55.997760763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.043495315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.044295775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.044350667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.044371304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.044406519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781} {"msg":"Crucible ce8e470a-9918-456e-977c-903bad519439 has session id: 7c5fbdaf-9b2d-4a39-be4e-86386ada168e","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.044985639Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Upstairs opts: Upstairs UUID: ce8e470a-9918-456e-977c-903bad519439, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.045015815Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Crucible stats registered with UUID: ce8e470a-9918-456e-977c-903bad519439","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.045030423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.07661699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781} {"msg":"ce8e470a-9918-456e-977c-903bad519439 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076657614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076676612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076706388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076727165Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076744552Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.07676133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076778248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076794586Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076821532Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076853187Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46668)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.076933006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54283)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.07698088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40474)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.077010686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.077819974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.077859128Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.077889484Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.07791819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.077987681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.078042673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.078553483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.078577819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.078601056Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079156139Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079183716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079199493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079274673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.07929743Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079311808Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079324336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079337634Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[0]R flush_numbers[0..12]: [256, 251, 260, 237, 260, 259, 251, 251, 250, 251, 251, 260]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079351542Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[0]R generation[0..12]: [147, 145, 149, 139, 149, 148, 145, 145, 144, 145, 145, 149]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.07936558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079379748Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[1]R flush_numbers[0..12]: [256, 251, 260, 237, 260, 259, 251, 251, 250, 251, 251, 260]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079393667Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[1]R generation[0..12]: [147, 145, 149, 139, 149, 148, 145, 145, 144, 145, 145, 149]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079407294Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079421223Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[2]R flush_numbers[0..12]: [256, 251, 260, 237, 260, 259, 251, 251, 250, 251, 251, 260]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079434931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[2]R generation[0..12]: [147, 145, 149, 139, 149, 148, 145, 145, 144, 145, 145, 149]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079448689Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079462587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079475375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079492643Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079506251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079519339Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"":"downstairs","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079532547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079544636Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"0","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-09-27T03:55:56.079557694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"1","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079579221Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"client":"2","":"downstairs","session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079592499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079604967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"ce8e470a-9918-456e-977c-903bad519439 is now active with session: 7c5fbdaf-9b2d-4a39-be4e-86386ada168e","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079617596Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079629744Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781,"session_id":"7c5fbdaf-9b2d-4a39-be4e-86386ada168e"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:55:56.079642172Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 566 len 20480 data: 14 13 14 14 12 02/30 Write block 559 len 4096 data: 13 03/30 Write block 352 len 4096 data: 19 04/30 Read block 335 len 32768 05/30 Read block 299 len 28672 06/30 Write block 225 len 36864 data: 13 16 17 18 16 16 13 9 10 07/30 Write block 153 len 16384 data: 10 8 7 8 08/30 Read block 574 len 8192 09/30 Read block 399 len 36864 10/30 Read block 41 len 16384 11/30 Read block 285 len 8192 12/30 Read block 343 len 12288 13/30 Read block 127 len 40960 14/30 Read block 261 len 4096 15/30 Write block 361 len 32768 data: 16 15 15 13 12 12 14 15 16/30 Flush 17/30 Flush 18/30 Write block 196 len 24576 data: 14 11 10 7 6 6 19/30 Write block 14 len 28672 data: 13 15 16 11 8 6 9 20/30 Read block 9 len 28672 21/30 Write block 278 len 36864 data: 13 13 12 12 10 11 10 11 12 22/30 Read block 538 len 24576 23/30 Read block 559 len 8192 24/30 Read block 384 len 4096 25/30 Read block 424 len 40960 26/30 Flush 27/30 Write block 396 len 40960 data: 11 10 10 10 9 9 11 11 12 13 28/30 Write block 276 len 36864 data: 13 13 14 14 13 13 11 12 11 29/30 Write block 333 len 20480 data: 11 12 12 11 12 30/30 Write block 131 len 40960 data: 17 17 15 14 10 12 11 13 15 13 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 10 Done Sent Done false 28 Acked 1027 Write 9 Sent Sent Done false 29 Acked 1028 Write 5 Sent Sent Sent false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 4 2 9 Done 1 0 2 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:55:56.105236907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1781} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 263 263 263 F F F 1 020-039 150 150 150 263 263 263 F F F 2 040-059 149 149 149 260 260 260 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 148 148 148 259 259 259 F F F 6 120-139 145 145 145 251 251 251 T T T 7 140-159 150 150 150 261 261 261 T T T 8 160-179 144 144 144 250 250 250 F F F 9 180-199 150 150 150 263 263 263 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 150 150 150 261 261 261 F F F 12 240-259 149 149 149 260 260 260 F F F 13 260-279 150 150 150 263 263 263 T T T 14 280-299 150 150 150 263 263 263 T T T 15 300-319 148 148 148 258 258 258 F F F 16 320-339 149 149 149 260 260 260 T T T 17 340-359 150 150 150 261 261 261 F F F 18 360-379 150 150 150 261 261 261 F F F 19 380-399 146 146 146 252 252 252 T T T 20 400-419 148 148 148 259 259 259 T T T 21 420-439 148 148 148 258 258 258 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 149 149 149 260 260 260 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 150 150 150 261 261 261 F F F 28 560-579 150 150 150 261 261 261 F F F 29 580-599 149 149 149 260 260 260 F F F Max gen: 150, Max flush: 263 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 263 263 263 F F F 1 020-039 150 150 150 263 263 263 F F F 2 040-059 149 149 149 260 260 260 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 148 148 148 259 259 259 F F F 6 120-139 151 151 151 264 264 264 F F F 7 140-159 151 151 151 264 264 264 F F F 8 160-179 144 144 144 250 250 250 F F F 9 180-199 150 150 150 263 263 263 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 150 150 150 261 261 261 F F F 12 240-259 149 149 149 260 260 260 F F F 13 260-279 151 151 151 264 264 264 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 148 148 148 258 258 258 F F F 16 320-339 151 151 151 264 264 264 F F F 17 340-359 150 150 150 261 261 261 F F F 18 360-379 150 150 150 261 261 261 F F F 19 380-399 151 151 151 264 264 264 F F F 20 400-419 151 151 151 264 264 264 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 142 142 142 245 245 245 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 149 149 149 260 260 260 F F F 26 520-539 142 142 142 242 242 242 F F F 27 540-559 150 150 150 261 261 261 F F F 28 560-579 150 150 150 261 261 261 F F F 29 580-599 149 149 149 260 260 260 F F F Max gen: 151, Max flush: 264 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.458526029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:00.502443493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.502994717Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.535354887Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.536105623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.536142138Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.536157046Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.536171324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789} {"msg":"Crucible 1bafca03-94a1-42a4-bc3e-1afd7b01a901 has session id: 0044c02e-cbbd-401b-bdff-922df12f7f33","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.536653017Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Upstairs opts: Upstairs UUID: 1bafca03-94a1-42a4-bc3e-1afd7b01a901, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.536700941Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Crucible stats registered with UUID: 1bafca03-94a1-42a4-bc3e-1afd7b01a901","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.536726487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568402902Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789} {"msg":"1bafca03-94a1-42a4-bc3e-1afd7b01a901 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568467483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568501898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568552651Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568592326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568629191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568662666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568699431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568734476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568765682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568797967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_connection connected from Ok(127.0.0.1:41383)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568836162Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_connection connected from Ok(127.0.0.1:42532)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568871787Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_connection connected from Ok(127.0.0.1:44981)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.568912871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.56964907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.569678066Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.569700372Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.569715301Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.569739867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.569767123Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.570381678Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.570409315Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.570424503Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571029679Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571065174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.57109274Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571110948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571125236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571142653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571155442Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.57116885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[0]R flush_numbers[0..12]: [263, 263, 260, 237, 260, 259, 264, 264, 250, 263, 263, 261]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571182758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[0]R generation[0..12]: [150, 150, 149, 139, 149, 148, 151, 151, 144, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571196946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571211254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[1]R flush_numbers[0..12]: [263, 263, 260, 237, 260, 259, 264, 264, 250, 263, 263, 261]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571225612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[1]R generation[0..12]: [150, 150, 149, 139, 149, 148, 151, 151, 144, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.57123933Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571253338Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[2]R flush_numbers[0..12]: [263, 263, 260, 237, 260, 259, 264, 264, 250, 263, 263, 261]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571266946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[2]R generation[0..12]: [150, 150, 149, 139, 149, 148, 151, 151, 144, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571280694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571303991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-09-27T03:56:00.571336557Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571358034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571375481Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571388669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"":"downstairs","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571401748Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571413666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"0","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571426784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"1","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571439952Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"client":"2","":"downstairs","session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.57145299Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571465349Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789region info: , block size: 4096 bytes "session_id":" sub_volume 0 blocks / extent: 20 0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"1bafca03-94a1-42a4-bc3e-1afd7b01a901 is now active with session: 0044c02e-cbbd-401b-bdff-922df12f7f33","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571484326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571496555Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789,"session_id":"0044c02e-cbbd-401b-bdff-922df12f7f33"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:00.571508763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 40960 data: 10 11 9 8 9 10 12 10 9 9 02/30 Write block 117 len 28672 data: 15 15 17 16 15 15 12 03/30 Write block 206 len 40960 data: 12 11 12 13 13 10 9 10 10 7 04/30 Read block 529 len 36864 05/30 Write block 390 len 4096 data: 15 06/30 Read block 164 len 8192 07/30 Flush 08/30 Write block 540 len 8192 data: 11 11 09/30 Read block 560 len 28672 10/30 Write block 19 len 40960 data: 7 10 9 12 10 8 4 5 7 8 11/30 Write block 255 len 28672 data: 8 9 11 10 9 11 13 12/30 Write block 468 len 16384 data: 9 9 10 12 13/30 Write block 100 len 28672 data: 14 12 10 6 6 7 8 14/30 Write block 147 len 28672 data: 12 11 11 9 9 11 12 15/30 Write block 48 len 24576 data: 15 18 15 16 13 12 16/30 Read block 505 len 20480 17/30 Read block 117 len 36864 18/30 Write block 531 len 28672 data: 14 13 13 13 12 13 10 19/30 Write block 440 len 24576 data: 6 6 6 5 5 4 20/30 Write block 180 len 16384 data: 10 14 13 14 21/30 Write block 223 len 40960 data: 10 13 14 17 18 19 17 17 14 10 22/30 Read block 135 len 16384 23/30 Flush 24/30 Write block 331 len 32768 data: 9 10 12 13 13 12 13 8 25/30 Write block 158 len 20480 data: 13 11 10 10 8 26/30 Read block 525 len 4096 27/30 Write block 566 len 20480 data: 15 14 15 15 13 28/30 Write block 558 len 16384 data: 14 14 11 11 29/30 Read block 274 len 40960 30/30 Read block 573 len 28672 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 4 Done Sent Done false 29 Acked 1028 Read 10 Done Sent Done false 30 Acked 1029 Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 3 0 3 Done 7 4 7 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:00.595618892Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1789} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 266 266 266 F F F 1 020-039 152 152 152 266 266 266 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 152 152 152 265 265 265 F F F 7 140-159 152 152 152 266 266 266 T T T 8 160-179 152 152 152 265 265 265 T T T 9 180-199 152 152 152 266 266 266 F F F 10 200-219 152 152 152 265 265 265 F F F 11 220-239 152 152 152 266 266 266 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 152 152 152 266 266 266 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 148 148 148 258 258 258 F F F 16 320-339 151 151 151 264 264 264 T T T 17 340-359 150 150 150 261 261 261 F F F 18 360-379 150 150 150 261 261 261 F F F 19 380-399 152 152 152 265 265 265 F F F 20 400-419 151 151 151 264 264 264 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 152 152 152 266 266 266 F F F 23 460-479 152 152 152 266 266 266 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 149 149 149 260 260 260 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 152 152 152 266 266 266 T T T 28 560-579 150 150 150 261 261 261 T T T 29 580-599 149 149 149 260 260 260 F F F Max gen: 152, Max flush: 266 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 266 266 266 F F F 1 020-039 152 152 152 266 266 266 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 152 152 152 265 265 265 F F F 7 140-159 153 153 153 267 267 267 F F F 8 160-179 153 153 153 267 267 267 F F F 9 180-199 152 152 152 266 266 266 F F F 10 200-219 152 152 152 265 265 265 F F F 11 220-239 152 152 152 266 266 266 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 152 152 152 266 266 266 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 148 148 148 258 258 258 F F F 16 320-339 153 153 153 267 267 267 F F F 17 340-359 150 150 150 261 261 261 F F F 18 360-379 150 150 150 261 261 261 F F F 19 380-399 152 152 152 265 265 265 F F F 20 400-419 151 151 151 264 264 264 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 152 152 152 266 266 266 F F F 23 460-479 152 152 152 266 266 266 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 149 149 149 260 260 260 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 153 153 153 267 267 267 F F F 28 560-579 153 153 153 267 267 267 F F F 29 580-599 149 149 149 260 260 260 F F F Max gen: 153, Max flush: 267 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.942619914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.987435853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.988289865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.98832644Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.988340868Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.988355116Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797} {"msg":"Crucible 3a5a2dca-e3dc-4fd8-91c1-14f0c2594dea has session id: df7bd7f6-3237-4d89-8394-548ee145604d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.98875793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Upstairs opts: Upstairs UUID: 3a5a2dca-e3dc-4fd8-91c1-14f0c2594dea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.988798495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Crucible stats registered with UUID: 3a5a2dca-e3dc-4fd8-91c1-14f0c2594dea","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:04.9888289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021108932Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797} {"msg":"3a5a2dca-e3dc-4fd8-91c1-14f0c2594dea active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021153046Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021171293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021211917Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021245583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021282168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021320792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021355247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021389583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021428877Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021466922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41602)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021507097Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36919)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021545911Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41057)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.021581416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.022410132Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.022442777Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.022463174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.022477752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.022578629Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.022597626Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023159438Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023196263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023309358Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023745517Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023779793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.02379814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023813978Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023830996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023847013Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023934551Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.023957038Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[0]R flush_numbers[0..12]: [266, 266, 266, 237, 260, 266, 265, 267, 267, 266, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.02401668Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[0]R generation[0..12]: [152, 152, 152, 139, 149, 152, 152, 153, 153, 152, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024043906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024059454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[1]R flush_numbers[0..12]: [266, 266, 266, 237, 260, 266, 265, 267, 267, 266, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024073632Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[1]R generation[0..12]: [152, 152, 152, 139, 149, 152, 152, 153, 153, 152, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.02408754Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024101928Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[2]R flush_numbers[0..12]: [266, 266, 266, 237, 260, 266, 265, 267, 267, 266, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024116296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[2]R generation[0..12]: [152, 152, 152, 139, 149, 152, 152, 153, 153, 152, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024130284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024144392Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024157531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024170669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024183567Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024196375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"":"downstairs","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"No downstairs reconciliation required","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-09-27T03:56:05.024218612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.0242309Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"0","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024244659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"1","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024257877Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"client":"2","":"downstairs","session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024271015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024283303Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"3a5a2dca-e3dc-4fd8-91c1-14f0c2594dea is now active with session: df7bd7f6-3237-4d89-8394-548ee145604d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024296191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.02430837Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797,"session_id":"df7bd7f6-3237-4d89-8394-548ee145604d"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:05.024320798Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 325 len 20480 data: 14 13 12 10 11 02/30 Read block 69 len 32768 03/30 Write block 332 len 36864 data: 11 13 14 14 13 14 9 7 8 04/30 Write block 598 len 4096 data: 6 05/30 Read block 233 len 8192 06/30 Write block 260 len 8192 data: 12 14 07/30 Read block 85 len 4096 08/30 Flush 09/30 Write block 376 len 36864 data: 17 20 14 19 17 14 15 16 13 10/30 Write block 196 len 12288 data: 15 12 11 11/30 Read block 514 len 16384 12/30 Read block 13 len 28672 13/30 Write block 449 len 20480 data: 11 9 8 9 10 14/30 Flush 15/30 Write block 553 len 32768 data: 12 13 13 14 11 15 15 12 16/30 Write block 175 len 28672 data: 7 8 10 11 11 11 15 17/30 Read block 539 len 4096 18/30 Write block 323 len 36864 data: 14 15 15 14 13 11 12 11 10 19/30 Read block 477 len 36864 20/30 Flush 21/30 Write block 267 len 4096 data: 10 22/30 Read block 549 len 36864 23/30 Read block 353 len 8192 24/30 Read block 188 len 32768 25/30 Write block 188 len 40960 data: 7 8 12 11 11 12 14 14 16 13 26/30 Write block 586 len 20480 data: 11 8 9 8 10 27/30 Read block 40 len 36864 28/30 Read block 326 len 8192 29/30 Read block 530 len 8192 30/30 Read block 550 len 12288 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:05.051003022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1797} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 266 266 266 F F F 1 020-039 152 152 152 266 266 266 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 152 152 152 265 265 265 F F F 7 140-159 153 153 153 267 267 267 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 154 154 154 270 270 270 T T T 10 200-219 152 152 152 265 265 265 F F F 11 220-239 152 152 152 266 266 266 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 154 154 154 268 268 268 T T T 14 280-299 151 151 151 264 264 264 F F F 15 300-319 148 148 148 258 258 258 F F F 16 320-339 154 154 154 270 270 270 F F F 17 340-359 154 154 154 268 268 268 F F F 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 151 151 151 264 264 264 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 154 154 154 269 269 269 F F F 23 460-479 152 152 152 266 266 266 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 149 149 149 260 260 260 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 154 154 154 270 270 270 F F F 28 560-579 154 154 154 270 270 270 F F F 29 580-599 154 154 154 268 268 268 T T T Max gen: 154, Max flush: 270 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 266 266 266 F F F 1 020-039 152 152 152 266 266 266 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 139 139 139 237 237 237 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 152 152 152 265 265 265 F F F 7 140-159 153 153 153 267 267 267 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 155 155 155 271 271 271 F F F 10 200-219 152 152 152 265 265 265 F F F 11 220-239 152 152 152 266 266 266 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 155 155 155 271 271 271 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 148 148 148 258 258 258 F F F 16 320-339 154 154 154 270 270 270 F F F 17 340-359 154 154 154 268 268 268 F F F 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 151 151 151 264 264 264 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 154 154 154 269 269 269 F F F 23 460-479 152 152 152 266 266 266 F F F 24 480-499 145 145 145 251 251 251 F F F 25 500-519 149 149 149 260 260 260 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 154 154 154 270 270 270 F F F 28 560-579 154 154 154 270 270 270 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 155, Max flush: 271 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.344131485Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.389883835Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.390613684Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.390660018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.390677806Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.390695823Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805} {"msg":"Crucible 122958a6-9536-4e9b-bf8c-0a7ee897ac8d has session id: 53d1391b-b45d-4e67-a1d9-6c48159fd696","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.391195784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Upstairs opts: Upstairs UUID: 122958a6-9536-4e9b-bf8c-0a7ee897ac8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.391245297Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Crucible stats registered with UUID: 122958a6-9536-4e9b-bf8c-0a7ee897ac8d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.391274093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.42228383Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805} {"msg":"122958a6-9536-4e9b-bf8c-0a7ee897ac8d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422320405Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422344152Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422372288Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422390175Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422407133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422423901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422440538Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422456746Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422473914Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422491171Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_connection connected from Ok(127.0.0.1:63920)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422581579Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_connection connected from Ok(127.0.0.1:48836)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422622283Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_connection connected from Ok(127.0.0.1:63971)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.422654289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.423524929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.423565453Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.423637723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.423670338Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.423703494Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.42373386Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424157591Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424185247Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424254858Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424709345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424737441Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424753369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424846566Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424879111Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424903898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424917686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424931514Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[0]R flush_numbers[0..12]: [266, 266, 266, 237, 260, 266, 265, 267, 270, 271, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424945732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[0]R generation[0..12]: [152, 152, 152, 139, 149, 152, 152, 153, 154, 155, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.42495971Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424977668Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[1]R flush_numbers[0..12]: [266, 266, 266, 237, 260, 266, 265, 267, 270, 271, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.424992156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[1]R generation[0..12]: [152, 152, 152, 139, 149, 152, 152, 153, 154, 155, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425006294Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425020352Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[2]R flush_numbers[0..12]: [266, 266, 266, 237, 260, 266, 265, 267, 270, 271, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.42503408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[2]R generation[0..12]: [152, 152, 152, 139, 149, 152, 152, 153, 154, 155, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425047858Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425061886Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425074954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425096261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:56:09.425121388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425147834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"":"downstairs","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425164122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.42517682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"0","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425190408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"1","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425206966Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"client":"2","":"downstairs","session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425220864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425233553Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"122958a6-9536-4e9b-bf8c-0a7ee897ac8d is now active with session: 53d1391b-b45d-4e67-a1d9-6c48159fd696","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425246561Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425258819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805,"session_id":"53d1391b-b45d-4e67-a1d9-6c48159fd696"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:09.425271237Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 36864 data: 9 5 6 8 9 9 10 10 9 02/30 Write block 73 len 28672 data: 18 13 12 14 15 13 12 03/30 Read block 452 len 32768 04/30 Read block 405 len 40960 05/30 Write block 400 len 28672 data: 10 10 12 12 13 14 14 06/30 Read block 502 len 28672 07/30 Write block 536 len 12288 data: 14 11 10 08/30 Write block 356 len 12288 data: 12 13 11 09/30 Write block 496 len 36864 data: 10 11 10 10 10 10 12 11 8 10/30 Read block 22 len 12288 11/30 Read block 57 len 20480 12/30 Write block 260 len 20480 data: 13 15 11 10 10 13/30 Read block 169 len 4096 14/30 Read block 73 len 16384 15/30 Read block 30 len 20480 16/30 Read block 540 len 20480 17/30 Write block 153 len 4096 data: 13 18/30 Write block 2 len 20480 data: 7 8 7 7 9 19/30 Read block 30 len 20480 20/30 Read block 400 len 8192 21/30 Write block 272 len 16384 data: 11 10 9 13 22/30 Read block 338 len 4096 23/30 Read block 327 len 12288 24/30 Write block 569 len 4096 data: 16 25/30 Write block 187 len 24576 data: 8 8 9 13 12 12 26/30 Write block 150 len 40960 data: 10 10 12 14 10 9 10 11 14 12 27/30 Write block 317 len 32768 data: 11 10 12 9 12 14 15 16 28/30 Write block 301 len 28672 data: 15 13 11 14 15 14 12 29/30 Read block 570 len 40960 30/30 Read block 410 len 20480 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 9 Done Done Done false 2 Acked 1001 Write 7 Done Done Done false 3 Acked 1002 Read 8 Done Done Done false 4 Acked 1003 Read 10 Done Done Done false 5 Acked 1004 Write 7 Done Done Done false 6 Acked 1005 Read 7 Done Done Done false 7 Acked 1006 Write 3 Done Done Done false 8 Acked 1007 Write 3 Done Done Done false 9 Acked 1008 Write 9 Done Done Done false 10 Acked 1009 Read 3 Done Done Done false 11 Acked 1010 Read 5 Done Done Done false 12 Acked 1011 Write 5 Done Done Done false 13 Acked 1012 Read 1 Done Done Done false 14 Acked 1013 Read 4 Done Done Done false 15 Acked 1014 Read 5 Done Done Done false 16 Acked 1015 Read 5 Done Done Done false 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:09.444525957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1805} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 266 266 266 T T T 1 020-039 152 152 152 266 266 266 T T T 2 040-059 152 152 152 266 266 266 F F F 3 060-079 139 139 139 237 237 237 T T T 4 080-099 149 149 149 260 260 260 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 152 152 152 265 265 265 F F F 7 140-159 153 153 153 267 267 267 T T T 8 160-179 154 154 154 270 270 270 F F F 9 180-199 155 155 155 271 271 271 T T T 10 200-219 152 152 152 265 265 265 F F F 11 220-239 152 152 152 266 266 266 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 155 155 155 271 271 271 T T T 14 280-299 151 151 151 264 264 264 F F F 15 300-319 148 148 148 258 258 258 T T T 16 320-339 154 154 154 270 270 270 T T T 17 340-359 154 154 154 268 268 268 T T T 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 151 151 151 264 264 264 T T T 21 420-439 148 148 148 258 258 258 F F F 22 440-459 154 154 154 269 269 269 F F F 23 460-479 152 152 152 266 266 266 F F F 24 480-499 145 145 145 251 251 251 T T T 25 500-519 149 149 149 260 260 260 T T T 26 520-539 152 152 152 266 266 266 T T T 27 540-559 154 154 154 270 270 270 F F F 28 560-579 154 154 154 270 270 270 T T T 29 580-599 155 155 155 271 271 271 F F F Max gen: 155, Max flush: 271 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 156 156 156 272 272 272 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 156 156 156 272 272 272 F F F 4 080-099 149 149 149 260 260 260 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 152 152 152 265 265 265 F F F 7 140-159 156 156 156 272 272 272 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 156 156 156 272 272 272 F F F 10 200-219 152 152 152 265 265 265 F F F 11 220-239 152 152 152 266 266 266 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 156 156 156 272 272 272 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 156 156 156 272 272 272 F F F 16 320-339 156 156 156 272 272 272 F F F 17 340-359 156 156 156 272 272 272 F F F 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 156 156 156 272 272 272 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 154 154 154 269 269 269 F F F 23 460-479 152 152 152 266 266 266 F F F 24 480-499 156 156 156 272 272 272 F F F 25 500-519 156 156 156 272 272 272 F F F 26 520-539 156 156 156 272 272 272 F F F 27 540-559 154 154 154 270 270 270 F F F 28 560-579 156 156 156 272 272 272 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 156, Max flush: 272 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.879505476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.925094599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.925866203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.925925794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.925944922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.925962879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813} {"msg":"Crucible 11a24c7c-84ae-4b50-aefc-001be83e5aa6 has session id: 9d5b2f25-c319-4eba-a308-ebe5b4ea9c18","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.926357335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Upstairs opts: Upstairs UUID: 11a24c7c-84ae-4b50-aefc-001be83e5aa6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.926403548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Crucible stats registered with UUID: 11a24c7c-84ae-4b50-aefc-001be83e5aa6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.926437234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958430534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813} {"msg":"11a24c7c-84ae-4b50-aefc-001be83e5aa6 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.95846332Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958480167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958507604Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958524931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958542179Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958558747Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958587653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.95860661Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958629407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958657143Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_connection connected from Ok(127.0.0.1:39191)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958858765Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_connection connected from Ok(127.0.0.1:49925)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958902879Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_connection connected from Ok(127.0.0.1:62317)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.958982938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.959573696Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.959610891Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.959726335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.959754302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.95976928Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.959783558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960251893Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960280839Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960454585Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960813155Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960837392Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.9608522Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960890255Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960904633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.960917791Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.9610659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961091127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[0]R flush_numbers[0..12]: [272, 272, 266, 272, 260, 266, 265, 272, 270, 272, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961109164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[0]R generation[0..12]: [156, 156, 152, 156, 149, 152, 152, 156, 154, 156, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961123472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961137491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[1]R flush_numbers[0..12]: [272, 272, 266, 272, 260, 266, 265, 272, 270, 272, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961152458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[1]R generation[0..12]: [156, 156, 152, 156, 149, 152, 152, 156, 154, 156, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961166287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961180154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[2]R flush_numbers[0..12]: [272, 272, 266, 272, 260, 266, 265, 272, 270, 272, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961193783Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[2]R generation[0..12]: [156, 156, 152, 156, 149, 152, 152, 156, 154, 156, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961207491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961229918Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961255684Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961281451Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:56:13.961306527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961331944Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"":"downstairs","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.96135738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961371028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"0","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961387616Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"1","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961411842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"client":"2","":"downstairs","session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961437589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961456876Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"11a24c7c-84ae-4b50-aefc-001be83e5aa6 is now active with session: 9d5b2f25-c319-4eba-a308-ebe5b4ea9c18","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961476564Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961510429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813,"session_id":"9d5b2f25-c319-4eba-a308-ebe5b4ea9c18"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:13.961536325Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 538 len 32768 data: 11 8 12 12 11 9 11 10 02/30 Read block 285 len 8192 03/30 Write block 493 len 8192 data: 8 7 04/30 Flush 05/30 Write block 220 len 8192 data: 10 14 06/30 Write block 187 len 36864 data: 9 9 10 14 13 13 13 15 15 07/30 Read block 308 len 16384 08/30 Write block 517 len 40960 data: 13 12 13 12 11 10 11 12 13 12 09/30 Read block 259 len 28672 10/30 Read block 470 len 20480 11/30 Write block 566 len 4096 data: 16 12/30 Read block 301 len 24576 13/30 Read block 342 len 20480 14/30 Write block 125 len 16384 data: 15 15 15 15 15/30 Read block 355 len 12288 16/30 Read block 145 len 12288 17/30 Write block 476 len 40960 data: 11 11 11 16 16 13 14 14 13 13 18/30 Flush 19/30 Write block 486 len 20480 data: 12 10 9 10 11 20/30 Read block 480 len 4096 21/30 Read block 182 len 12288 22/30 Read block 260 len 28672 23/30 Read block 72 len 32768 24/30 Read block 312 len 12288 25/30 Read block 395 len 4096 26/30 Write block 468 len 28672 data: 10 10 11 13 12 9 11 27/30 Read block 281 len 32768 28/30 Write block 192 len 32768 data: 14 14 16 16 17 14 12 8 29/30 Write block 87 len 36864 data: 12 14 13 12 11 10 9 7 9 30/30 Read block 177 len 36864 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 11 12 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:13.986998608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1813} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 156 156 156 272 272 272 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 156 156 156 272 272 272 F F F 4 080-099 149 149 149 260 260 260 T T T 5 100-119 152 152 152 266 266 266 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 156 156 156 272 272 272 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 158 158 158 274 274 274 T T T 10 200-219 152 152 152 265 265 265 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 156 156 156 272 272 272 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 156 156 156 272 272 272 F F F 16 320-339 156 156 156 272 272 272 F F F 17 340-359 156 156 156 272 272 272 F F F 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 156 156 156 272 272 272 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 154 154 154 269 269 269 F F F 23 460-479 158 158 158 274 274 274 T T T 24 480-499 158 158 158 274 274 274 T T T 25 500-519 158 158 158 274 274 274 F F F 26 520-539 158 158 158 274 274 274 F F F 27 540-559 158 158 158 273 273 273 F F F 28 560-579 158 158 158 274 274 274 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 158, Max flush: 274 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 156 156 156 272 272 272 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 156 156 156 272 272 272 F F F 4 080-099 159 159 159 275 275 275 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 156 156 156 272 272 272 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 159 159 159 275 275 275 F F F 10 200-219 152 152 152 265 265 265 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 152 152 152 266 266 266 F F F 13 260-279 156 156 156 272 272 272 F F F 14 280-299 151 151 151 264 264 264 F F F 15 300-319 156 156 156 272 272 272 F F F 16 320-339 156 156 156 272 272 272 F F F 17 340-359 156 156 156 272 272 272 F F F 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 156 156 156 272 272 272 F F F 21 420-439 148 148 148 258 258 258 F F F 22 440-459 154 154 154 269 269 269 F F F 23 460-479 159 159 159 275 275 275 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 158 158 158 274 274 274 F F F 26 520-539 158 158 158 274 274 274 F F F 27 540-559 158 158 158 273 273 273 F F F 28 560-579 158 158 158 274 274 274 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 159, Max flush: 275 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.285343681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.330988716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.331796464Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.331834969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.331856396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.331872084Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821} {"msg":"Crucible 33fe1eb5-86e9-4552-b313-ebcbf8413b8d has session id: 72de82fc-8cfe-4ea8-addd-0f0558de7271","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.332274148Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Upstairs opts: Upstairs UUID: 33fe1eb5-86e9-4552-b313-ebcbf8413b8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.332316472Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Crucible stats registered with UUID: 33fe1eb5-86e9-4552-b313-ebcbf8413b8d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.332341669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363686449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821} {"msg":"33fe1eb5-86e9-4552-b313-ebcbf8413b8d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363734423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.36375422Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363782196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363800803Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363818081Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363834479Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363856176Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363886392Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363916558Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363947693Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_connection connected from Ok(127.0.0.1:46969)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.363981618Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_connection connected from Ok(127.0.0.1:33671)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.364012784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_connection connected from Ok(127.0.0.1:58267)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.364053449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.36505719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365089056Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365105993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365124521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365139499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365153167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365758083Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365797728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.365835952Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366363009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366389576Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366404394Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366500921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366533006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366556673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366570061Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366583249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[0]R flush_numbers[0..12]: [272, 272, 266, 272, 275, 266, 274, 272, 270, 275, 265, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366596847Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[0]R generation[0..12]: [156, 156, 152, 156, 159, 152, 158, 156, 154, 159, 152, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366614775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366628963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[1]R flush_numbers[0..12]: [272, 272, 266, 272, 275, 266, 274, 272, 270, 275, 265, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366642721Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[1]R generation[0..12]: [156, 156, 152, 156, 159, 152, 158, 156, 154, 159, 152, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366656399Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366671347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[2]R flush_numbers[0..12]: [272, 272, 266, 272, 275, 266, 274, 272, 270, 275, 265, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366685095Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[2]R generation[0..12]: [156, 156, 152, 156, 159, 152, 158, 156, 154, 159, 152, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366698933Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366712851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366725739Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366741277Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366764884Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366824196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"":"downstairs","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366848853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366870999Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"0","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366888937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"1","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366903355Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"client":"2","":"downstairs","session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366916643Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366931711Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"33fe1eb5-86e9-4552-b313-ebcbf8413b8d is now active with session: 72de82fc-8cfe-4ea8-addd-0f0558de7271","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366954698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366970986Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821,"session_id":"72de82fc-8cfe-4ea8-addd-0f0558de7271"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:18.366983774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 32768 data: 11 10 11 13 15 16 18 16 02/30 Read block 489 len 40960 03/30 Read block 547 len 8192 04/30 Write block 202 len 24576 data: 7 9 9 12 13 12 05/30 Write block 458 len 16384 data: 17 19 17 16 06/30 Read block 586 len 32768 07/30 Read block 268 len 32768 08/30 Read block 459 len 20480 09/30 Flush 10/30 Write block 350 len 16384 data: 19 20 20 16 11/30 Read block 572 len 28672 12/30 Read block 486 len 8192 13/30 Write block 311 len 20480 data: 8 9 8 8 8 14/30 Read block 197 len 32768 15/30 Write block 38 len 8192 data: 12 13 16/30 Read block 596 len 16384 17/30 Read block 277 len 16384 18/30 Write block 543 len 28672 data: 10 12 11 11 10 10 9 19/30 Read block 160 len 40960 20/30 Read block 488 len 32768 21/30 Write block 272 len 40960 data: 12 11 10 14 14 14 15 15 14 14 22/30 Write block 143 len 28672 data: 11 10 12 13 13 12 12 23/30 Write block 70 len 24576 data: 20 20 19 19 14 13 24/30 Flush 25/30 Write block 424 len 12288 data: 8 10 12 26/30 Write block 319 len 8192 data: 13 10 27/30 Write block 21 len 32768 data: 10 13 11 10 6 7 9 10 28/30 Write block 517 len 8192 data: 14 13 29/30 Write block 94 len 24576 data: 8 10 9 9 11 13 30/30 Write block 247 len 24576 data: 23 21 20 16 13 12 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Write 8 Sent Done Done false 28 Acked 1027 Write 2 Sent Sent Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 2 9 Done 2 3 4 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:18.391281047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1821} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 160 160 160 277 277 277 T T T 2 040-059 152 152 152 266 266 266 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 159 159 159 275 275 275 F T T <--- 5 100-119 152 152 152 266 266 266 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 160 160 160 277 277 277 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 159 159 159 275 275 275 F F F 10 200-219 160 160 160 276 276 276 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 152 152 152 266 266 266 F T T <--- 13 260-279 160 160 160 277 277 277 F F F 14 280-299 160 160 160 277 277 277 F F F 15 300-319 160 160 160 277 277 277 T T T 16 320-339 156 156 156 272 272 272 T T T 17 340-359 160 160 160 277 277 277 F F F 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 156 156 156 272 272 272 F F F 21 420-439 148 148 148 258 258 258 T T T 22 440-459 160 160 160 276 276 276 F F F 23 460-479 160 160 160 276 276 276 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 158 158 158 274 274 274 T T T 26 520-539 158 158 158 274 274 274 F F F 27 540-559 160 160 160 277 277 277 F F F 28 560-579 158 158 158 274 274 274 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 160, Max flush: 277 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 161 161 161 278 278 278 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 160 160 160 277 277 277 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 159 159 159 275 275 275 F F F 10 200-219 160 160 160 276 276 276 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 161 161 161 278 278 278 F F F 13 260-279 160 160 160 277 277 277 F F F 14 280-299 160 160 160 277 277 277 F F F 15 300-319 161 161 161 278 278 278 F F F 16 320-339 161 161 161 278 278 278 F F F 17 340-359 160 160 160 277 277 277 F F F 18 360-379 154 154 154 269 269 269 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 156 156 156 272 272 272 F F F 21 420-439 161 161 161 278 278 278 F F F 22 440-459 160 160 160 276 276 276 F F F 23 460-479 160 160 160 276 276 276 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 161 161 161 278 278 278 F F F 26 520-539 158 158 158 274 274 274 F F F 27 540-559 160 160 160 277 277 277 F F F 28 560-579 158 158 158 274 274 274 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 161, Max flush: 278 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.733677355Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.779930726Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.780861967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.780900852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.78091544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.780930448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829} {"msg":"Crucible 273dee6c-06fc-4f1c-b06b-5c3e3a964f56 has session id: 5614f2be-35e6-4c0a-920d-ad48ec7b45bf","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.781385595Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Upstairs opts: Upstairs UUID: 273dee6c-06fc-4f1c-b06b-5c3e3a964f56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.781427499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Crucible stats registered with UUID: 273dee6c-06fc-4f1c-b06b-5c3e3a964f56","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.781452036Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813246264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829} {"msg":"273dee6c-06fc-4f1c-b06b-5c3e3a964f56 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813306175Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813338851Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813384365Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813428119Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813462784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813499199Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813533444Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81356689Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813601265Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81363304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:54471)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813665416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:45763)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813698191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:50805)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.813839972Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81450324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.814533566Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.814549804Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.814563662Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.814643741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.814664298Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81522848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.815332286Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.815366561Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81580474Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.815830457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.815844915Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.815911216Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81595053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.815979606Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816035969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816067494Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[0]R flush_numbers[0..12]: [272, 278, 266, 277, 278, 266, 274, 277, 270, 275, 276, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816089491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[0]R generation[0..12]: [156, 161, 152, 160, 161, 152, 158, 160, 154, 159, 160, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816105109Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816119887Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[1]R flush_numbers[0..12]: [272, 278, 266, 277, 278, 266, 274, 277, 270, 275, 276, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816134075Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[1]R generation[0..12]: [156, 161, 152, 160, 161, 152, 158, 160, 154, 159, 160, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816152042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81616681Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[2]R flush_numbers[0..12]: [272, 278, 266, 277, 278, 266, 274, 277, 270, 275, 276, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816181088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[2]R generation[0..12]: [156, 161, 152, 160, 161, 152, 158, 160, 154, 159, 160, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816194987Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816208905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816221703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816234901Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816247609Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816260407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"":"downstairs","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816273346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:56:22.816285304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"0","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816304841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"1","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816317899Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"client":"2","":"downstairs","session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816331038Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816346335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"273dee6c-06fc-4f1c-b06b-5c3e3a964f56 is now active with session: 5614f2be-35e6-4c0a-920d-ad48ec7b45bf","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816359424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.816371402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829,"session_id":"5614f2be-35e6-4c0a-920d-ad48ec7b45bf"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:22.81638345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 526 len 36864 data: 13 11 13 12 14 15 14 14 14 02/30 Write block 443 len 28672 data: 6 6 5 5 7 10 12 03/30 Read block 488 len 32768 04/30 Read block 125 len 36864 05/30 Read block 327 len 40960 06/30 Write block 328 len 4096 data: 12 07/30 Flush 08/30 Write block 124 len 4096 data: 13 09/30 Write block 251 len 4096 data: 14 10/30 Write block 275 len 12288 data: 15 15 15 11/30 Read block 597 len 8192 12/30 Read block 390 len 4096 13/30 Flush 14/30 Write block 401 len 4096 data: 11 15/30 Write block 203 len 20480 data: 10 10 13 14 13 16/30 Read block 290 len 28672 17/30 Write block 571 len 4096 data: 12 18/30 Write block 362 len 32768 data: 16 16 14 13 13 15 16 15 19/30 Flush 20/30 Write block 289 len 36864 data: 9 9 9 10 12 10 10 14 16 21/30 Write block 469 len 16384 data: 11 12 14 13 22/30 Flush 23/30 Write block 514 len 20480 data: 14 15 14 15 14 24/30 Write block 188 len 32768 data: 10 11 15 14 15 15 17 17 25/30 Read block 378 len 36864 26/30 Write block 463 len 32768 data: 15 15 17 13 11 11 12 13 27/30 Write block 125 len 32768 data: 16 16 16 16 15 17 18 18 28/30 Write block 206 len 16384 data: 15 14 13 14 29/30 Write block 531 len 12288 data: 16 15 15 30/30 Write block 412 len 12288 data: 12 10 11 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 8 Sent Sent Done false 28 Acked 1027 Write 4 Sent Sent Sent false 29 Acked 1028 Write 3 Sent Sent Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 3 11 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:22.841041464Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1829} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 161 161 161 278 278 278 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 162 162 162 280 280 280 T T T 7 140-159 160 160 160 277 277 277 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 159 159 159 275 275 275 T T T 10 200-219 162 162 162 281 281 281 T T T 11 220-239 158 158 158 274 274 274 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 282 282 282 F F F 15 300-319 161 161 161 278 278 278 F F F 16 320-339 162 162 162 279 279 279 F F F 17 340-359 160 160 160 277 277 277 F F F 18 360-379 162 162 162 281 281 281 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 162 162 162 281 281 281 T T F <--- 21 420-439 161 161 161 278 278 278 F F F 22 440-459 162 162 162 279 279 279 F F F 23 460-479 162 162 162 282 282 282 T T T 24 480-499 159 159 159 275 275 275 F F F 25 500-519 161 161 161 278 278 278 T T T 26 520-539 162 162 162 279 279 279 T T T 27 540-559 160 160 160 277 277 277 F F F 28 560-579 162 162 162 281 281 281 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 162, Max flush: 282 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 161 161 161 278 278 278 F F F 2 040-059 152 152 152 266 266 266 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 163 163 163 283 283 283 F F F 7 140-159 160 160 160 277 277 277 F F F 8 160-179 154 154 154 270 270 270 F F F 9 180-199 163 163 163 283 283 283 F F F 10 200-219 163 163 163 283 283 283 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 282 282 282 F F F 15 300-319 161 161 161 278 278 278 F F F 16 320-339 162 162 162 279 279 279 F F F 17 340-359 160 160 160 277 277 277 F F F 18 360-379 162 162 162 281 281 281 F F F 19 380-399 154 154 154 269 269 269 F F F 20 400-419 163 163 163 283 283 283 F F F 21 420-439 161 161 161 278 278 278 F F F 22 440-459 162 162 162 279 279 279 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 163 163 163 283 283 283 F F F 26 520-539 163 163 163 283 283 283 F F F 27 540-559 160 160 160 277 277 277 F F F 28 560-579 162 162 162 281 281 281 F F F 29 580-599 155 155 155 271 271 271 F F F Max gen: 163, Max flush: 283 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.190130427Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.236610177Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.23731173Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.237349155Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.237363813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.237378431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837} {"msg":"Crucible a2956262-667c-4826-9d1d-7ecebf6f5da5 has session id: 8af860b0-f226-4ad0-8c87-17059a437998","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.237855505Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Upstairs opts: Upstairs UUID: a2956262-667c-4826-9d1d-7ecebf6f5da5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.237894419Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Crucible stats registered with UUID: a2956262-667c-4826-9d1d-7ecebf6f5da5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.237911217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269290792Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837} {"msg":"a2956262-667c-4826-9d1d-7ecebf6f5da5 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269333856Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269351854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.26937909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269396518Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269413306Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269429843Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.26945348Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269471087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269488285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269505303Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_connection connected from Ok(127.0.0.1:35733)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269663841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_connection connected from Ok(127.0.0.1:50822)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.269706225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_connection connected from Ok(127.0.0.1:40140)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.26974045Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.270577794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.270616249Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.270635047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.270649795Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.270778937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.270855686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.271290936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.271319642Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.27140926Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.271919459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.271947755Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.271963203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.271976041Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.27198957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272002998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272015356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272028534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[0]R flush_numbers[0..12]: [272, 278, 266, 277, 278, 266, 283, 277, 270, 283, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272042402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[0]R generation[0..12]: [156, 161, 152, 160, 161, 152, 163, 160, 154, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.27205636Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272070498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[1]R flush_numbers[0..12]: [272, 278, 266, 277, 278, 266, 283, 277, 270, 283, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272084286Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[1]R generation[0..12]: [156, 161, 152, 160, 161, 152, 163, 160, 154, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272097935Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272111943Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[2]R flush_numbers[0..12]: [272, 278, 266, 277, 278, 266, 283, 277, 270, 283, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.27212962Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[2]R generation[0..12]: [156, 161, 152, 160, 161, 152, 163, 160, 154, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272144068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272158116Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272183533Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272210109Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272230376Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272254153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"":"downstairs","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272278819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272299347Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"0","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272313945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"1","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272327173Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"client":"2","":"downstairs","session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272340401Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible"region info: ," block size: 4096 bytes level":30 sub_volume 0 blocks / extent: 20 ,"time":"2024-09-27T03:56:27.272356849Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"a2956262-667c-4826-9d1d-7ecebf6f5da5 is now active with session: 8af860b0-f226-4ad0-8c87-17059a437998","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272370407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272385185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837,"session_id":"8af860b0-f226-4ad0-8c87-17059a437998"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:27.272397903Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 579 len 4096 data: 12 02/30 Write block 304 len 8192 data: 15 16 03/30 Read block 158 len 32768 04/30 Write block 41 len 32768 data: 14 15 16 15 17 18 15 16 05/30 Write block 374 len 32768 data: 10 13 18 21 15 20 18 15 06/30 Read block 471 len 20480 07/30 Flush 08/30 Write block 161 len 4096 data: 11 09/30 Write block 133 len 4096 data: 16 10/30 Read block 448 len 36864 11/30 Read block 160 len 16384 12/30 Read block 425 len 12288 13/30 Flush 14/30 Write block 591 len 20480 data: 10 10 8 8 8 15/30 Read block 187 len 12288 16/30 Read block 542 len 12288 17/30 Write block 404 len 12288 data: 14 15 15 18/30 Read block 241 len 12288 19/30 Read block 550 len 36864 20/30 Write block 154 len 4096 data: 11 21/30 Read block 157 len 12288 22/30 Write block 530 len 28672 data: 15 17 16 16 15 13 15 23/30 Write block 333 len 4096 data: 14 24/30 Write block 508 len 8192 data: 6 7 25/30 Read block 565 len 32768 26/30 Read block 481 len 36864 27/30 Write block 372 len 20480 data: 12 10 11 14 19 28/30 Read block 493 len 24576 29/30 Write block 375 len 32768 data: 15 20 22 16 21 19 16 16 30/30 Read block 365 len 4096 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 5 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 3 Done Done Done false 17 Acked 1016 Write 3 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Read 9 Done Done Done false 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 8 Sent Done Sent false 30 Acked 1029 Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 15 17 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:27.294336153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1837} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 161 161 161 278 278 278 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 164 164 164 285 285 285 F F F 7 140-159 160 160 160 277 277 277 T T T 8 160-179 164 164 164 285 285 285 F F F 9 180-199 163 163 163 283 283 283 F F F 10 200-219 163 163 163 283 283 283 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 282 282 282 F F F 15 300-319 164 164 164 284 284 284 F F F 16 320-339 162 162 162 279 279 279 T T T 17 340-359 160 160 160 277 277 277 F F F 18 360-379 164 164 164 284 284 284 T T T 19 380-399 164 164 164 284 284 284 T T T 20 400-419 163 163 163 283 283 283 T T T 21 420-439 161 161 161 278 278 278 F F F 22 440-459 162 162 162 279 279 279 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 163 163 163 283 283 283 T T T 26 520-539 163 163 163 283 283 283 T T T 27 540-559 160 160 160 277 277 277 F F F 28 560-579 164 164 164 284 284 284 F F F 29 580-599 155 155 155 271 271 271 T T T Max gen: 164, Max flush: 285 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 161 161 161 278 278 278 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 164 164 164 285 285 285 F F F 7 140-159 165 165 165 286 286 286 F F F 8 160-179 164 164 164 285 285 285 F F F 9 180-199 163 163 163 283 283 283 F F F 10 200-219 163 163 163 283 283 283 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 162 162 162 280 280 280 F F F 14 280-299 162 162 162 282 282 282 F F F 15 300-319 164 164 164 284 284 284 F F F 16 320-339 165 165 165 286 286 286 F F F 17 340-359 160 160 160 277 277 277 F F F 18 360-379 165 165 165 286 286 286 F F F 19 380-399 165 165 165 286 286 286 F F F 20 400-419 165 165 165 286 286 286 F F F 21 420-439 161 161 161 278 278 278 F F F 22 440-459 162 162 162 279 279 279 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 165 165 165 286 286 286 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 160 160 160 277 277 277 F F F 28 560-579 164 164 164 284 284 284 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 165, Max flush: 286 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.650446316Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.69515491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.695906486Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.69595264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.695969848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.695987485Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845} {"msg":"Crucible 52149bbf-87bb-4f52-858e-6cc607c647ea has session id: cb2b0d80-2271-46cd-b159-cdf713df3f64","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.696415556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Upstairs opts: Upstairs UUID: 52149bbf-87bb-4f52-858e-6cc607c647ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.696455011Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Crucible stats registered with UUID: 52149bbf-87bb-4f52-858e-6cc607c647ea","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.696476168Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728004673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845} {"msg":"52149bbf-87bb-4f52-858e-6cc607c647ea active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728064115Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.72809845Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728144813Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728182848Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728216913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728249619Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728279975Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728306151Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"io task","client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728324988Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"io task","client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728342996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"io task","client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_connection connected from Ok(127.0.0.1:64521)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728361673Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"io task","client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_connection connected from Ok(127.0.0.1:37889)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.72838398Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"io task","client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_connection connected from Ok(127.0.0.1:37836)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.728411356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"io task","client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.729299924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.729333269Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.729348217Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.729362065Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.72947181Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.729517864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.729955463Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730041521Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730061499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730553471Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730580447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730594795Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730677594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730698451Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730711969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730724307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730737605Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[0]R flush_numbers[0..12]: [272, 278, 284, 277, 278, 266, 285, 286, 285, 283, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730751263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[0]R generation[0..12]: [156, 161, 164, 160, 161, 152, 164, 165, 164, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730765191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.73077906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[1]R flush_numbers[0..12]: [272, 278, 284, 277, 278, 266, 285, 286, 285, 283, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730792827Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[1]R generation[0..12]: [156, 161, 164, 160, 161, 152, 164, 165, 164, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730806736Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730820504Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[2]R flush_numbers[0..12]: [272, 278, 284, 277, 278, 266, 285, 286, 285, 283, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730838041Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[2]R generation[0..12]: [156, 161, 164, 160, 161, 152, 164, 165, 164, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730852029Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730866957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730880086Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730893154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730907192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.73092022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"":"downstairs","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730933368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730945277Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"0","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730958465Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"1","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.730971473Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"client":"2","":"downstairs","session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.73099253Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.731005148Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"52149bbf-87bb-4f52-858e-6cc607c647ea is now active with session: cb2b0d80-2271-46cd-b159-cdf713df3f64","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.731018007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.731030005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845,"session_id":"cb2b0d80-2271-46cd-b159-cdf713df3f64"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:31.731042193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 281 len 28672 data: 15 12 13 12 12 13 11 02/30 Write block 154 len 40960 data: 12 10 11 12 15 13 11 12 9 7 03/30 Write block 266 len 8192 data: 10 11 04/30 Flush 05/30 Write block 416 len 16384 data: 10 13 16 13 06/30 Read block 172 len 8192 07/30 Read block 373 len 40960 08/30 Read block 303 len 20480 09/30 Read block 255 len 12288 10/30 Read block 263 len 16384 11/30 Read block 353 len 16384 12/30 Read block 340 len 20480 13/30 Write block 323 len 40960 data: 16 17 16 15 14 13 13 12 11 12 14/30 Read block 474 len 16384 15/30 Read block 180 len 16384 16/30 Read block 357 len 8192 17/30 Read block 274 len 12288 18/30 Read block 291 len 20480 19/30 Read block 65 len 16384 20/30 Read block 236 len 32768 21/30 Read block 559 len 8192 22/30 Write block 28 len 24576 data: 11 10 11 11 10 11 23/30 Write block 554 len 24576 data: 14 14 15 12 16 16 24/30 Read block 446 len 16384 25/30 Read block 131 len 24576 26/30 Read block 210 len 24576 27/30 Write block 415 len 8192 data: 10 11 28/30 Write block 314 len 28672 data: 9 9 9 12 11 14 11 29/30 Write block 181 len 40960 data: 16 14 15 12 10 11 10 11 12 16 30/30 Read block 265 len 8192 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 5 Acked 1004 Write 4 Done Done Done false 6 Acked 1005 Read 2 Done Done Done false 7 Acked 1006 Read 10 Done Done Done false 8 Acked 1007 Read 5 Done Done Done false 9 Acked 1008 Read 3 Done Done Done false 10 Acked 1009 Read 4 Done Done Done false 11 Acked 1010 Read 4 Done Done Done false 12 Acked 1011 Read 5 Done Done Done false 13 Acked 1012 Write 10 Done Done Done false 14 Acked 1013 Read 4 Done Done Done false 15 Acked 1014 Read 4 Done Done Done false 16 Acked 1015 Read 2 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Read 4 Done Done Done false 20 Acked 1019 Read 8 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:31.753954518Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1845} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 161 161 161 278 278 278 T T T 2 040-059 164 164 164 284 284 284 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 164 164 164 285 285 285 F F F 7 140-159 166 166 166 287 287 287 F F F 8 160-179 166 166 166 287 287 287 F F F 9 180-199 163 163 163 283 283 283 T T T 10 200-219 163 163 163 283 283 283 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 166 166 166 287 287 287 F F F 14 280-299 166 166 166 287 287 287 F F F 15 300-319 164 164 164 284 284 284 T T T 16 320-339 165 165 165 286 286 286 T T T 17 340-359 160 160 160 277 277 277 F F F 18 360-379 165 165 165 286 286 286 F F F 19 380-399 165 165 165 286 286 286 F F F 20 400-419 165 165 165 286 286 286 T T T 21 420-439 161 161 161 278 278 278 F F F 22 440-459 162 162 162 279 279 279 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 165 165 165 286 286 286 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 160 160 160 277 277 277 T T T 28 560-579 164 164 164 284 284 284 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 166, Max flush: 287 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 167 167 167 288 288 288 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 F F F 6 120-139 164 164 164 285 285 285 F F F 7 140-159 166 166 166 287 287 287 F F F 8 160-179 166 166 166 287 287 287 F F F 9 180-199 167 167 167 288 288 288 F F F 10 200-219 163 163 163 283 283 283 F F F 11 220-239 158 158 158 274 274 274 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 166 166 166 287 287 287 F F F 14 280-299 166 166 166 287 287 287 F F F 15 300-319 167 167 167 288 288 288 F F F 16 320-339 167 167 167 288 288 288 F F F 17 340-359 160 160 160 277 277 277 F F F 18 360-379 165 165 165 286 286 286 F F F 19 380-399 165 165 165 286 286 286 F F F 20 400-419 167 167 167 288 288 288 F F F 21 420-439 161 161 161 278 278 278 F F F 22 440-459 162 162 162 279 279 279 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 165 165 165 286 286 286 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 167 167 167 288 288 288 F F F 28 560-579 164 164 164 284 284 284 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 167, Max flush: 288 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.102413348Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.148482224Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.149467578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.149503753Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.149519261Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.149533299Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853} {"msg":"Crucible 5d463980-d26a-44a8-a9f3-8fa3090327ee has session id: 6115ce1e-273a-414b-bf75-868f28b1b3f6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.150065615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Upstairs opts: Upstairs UUID: 5d463980-d26a-44a8-a9f3-8fa3090327ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.15010619Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Crucible stats registered with UUID: 5d463980-d26a-44a8-a9f3-8fa3090327ee","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.150128327Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182473559Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853} {"msg":"5d463980-d26a-44a8-a9f3-8fa3090327ee active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182524712Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182543049Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182574465Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182591992Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.1826106Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182651464Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182691659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182720005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.18275093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182785336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48908)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182819781Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51019)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.182869524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57458)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.183068996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.183597953Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.183629259Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.183644657Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.183665864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184066329Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184109153Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184278529Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184404242Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184724658Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.18485137Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184870938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184884706Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184949617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184968654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.184982202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185233088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185257654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[0]R flush_numbers[0..12]: [272, 288, 284, 277, 278, 266, 285, 287, 287, 288, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185273462Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[0]R generation[0..12]: [156, 167, 164, 160, 161, 152, 164, 166, 166, 167, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.18528771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185301838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[1]R flush_numbers[0..12]: [272, 288, 284, 277, 278, 266, 285, 287, 287, 288, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185315546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[1]R generation[0..12]: [156, 167, 164, 160, 161, 152, 164, 166, 166, 167, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185329154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185346132Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[2]R flush_numbers[0..12]: [272, 288, 284, 277, 278, 266, 285, 287, 287, 288, 283, 274]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.18536015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[2]R generation[0..12]: [156, 167, 164, 160, 161, 152, 164, 166, 166, 167, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185373838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185387796Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185400584Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185413823Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185426871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185439739Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"":"downstairs","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185453167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185465125Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"0","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185484583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"1","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185498101Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"client":"2","":"downstairs","session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185511669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185524107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"5d463980-d26a-44a8-a9f3-8fa3090327ee is now active with session: 6115ce1e-273a-414b-bf75-868f28b1b3f6","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185536775Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185550834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853,"session_id":"6115ce1e-273a-414b-bf75-868f28b1b3f6"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:36.185563172Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 103 len 24576 data: 7 7 8 9 10 11 02/30 Write block 172 len 32768 data: 4 5 8 8 9 11 12 12 03/30 Write block 433 len 8192 data: 8 8 04/30 Write block 177 len 28672 data: 12 13 13 12 17 15 16 05/30 Write block 203 len 4096 data: 11 06/30 Write block 440 len 40960 data: 7 7 7 7 7 6 6 8 11 13 07/30 Read block 264 len 16384 08/30 Read block 149 len 8192 09/30 Write block 223 len 36864 data: 11 14 15 18 19 20 18 18 15 10/30 Read block 377 len 36864 11/30 Read block 247 len 28672 12/30 Read block 542 len 16384 13/30 Read block 341 len 40960 14/30 Read block 593 len 16384 15/30 Write block 514 len 20480 data: 15 16 15 16 15 16/30 Read block 339 len 32768 17/30 Read block 510 len 20480 18/30 Read block 400 len 36864 19/30 Write block 340 len 20480 data: 9 11 11 8 10 20/30 Read block 364 len 40960 21/30 Write block 231 len 16384 data: 16 11 11 12 22/30 Write block 121 len 32768 data: 16 16 13 14 17 17 17 17 23/30 Read block 212 len 8192 24/30 Write block 353 len 12288 data: 17 13 13 25/30 Write block 453 len 20480 data: 12 11 12 14 16 26/30 Write block 154 len 28672 data: 13 11 12 13 16 14 12 27/30 Read block 161 len 24576 28/30 Write block 411 len 24576 data: 14 13 11 12 11 12 29/30 Read block 526 len 28672 30/30 Read block 293 len 12288 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Write 8 Done Done Done false 3 Acked 1002 Write 2 Done Done Done false 4 Acked 1003 Write 7 Done Done Done false 5 Acked 1004 Write 1 Done Done Done false 6 Acked 1005 Write 10 Done Done Done false 7 Acked 1006 Read 4 Done Done Done false 8 Acked 1007 Read 2 Done Done Done false 9 Acked 1008 Write 9 Done Done Done false 10 Acked 1009 Read 9 Done Done Done false 11 Acked 1010 Read 7 Done Done Done false 12 Acked 1011 Read 4 Done Done Done false 13 Acked 1012 Read 10 Done Done Done false 14 Acked 1013 Read 4 Done Done Done false 15 Acked 1014 Write 5 Done Done Done false 16 Acked 1015 Read 8 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 29 30 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:36.205717388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1853} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 167 167 167 288 288 288 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 152 152 152 266 266 266 T T T 6 120-139 164 164 164 285 285 285 T T T 7 140-159 166 166 166 287 287 287 T T T 8 160-179 166 166 166 287 287 287 T T T 9 180-199 167 167 167 288 288 288 T T T 10 200-219 163 163 163 283 283 283 T T T 11 220-239 158 158 158 274 274 274 T T T 12 240-259 162 162 162 280 280 280 F F F 13 260-279 166 166 166 287 287 287 F F F 14 280-299 166 166 166 287 287 287 F F F 15 300-319 167 167 167 288 288 288 F F F 16 320-339 167 167 167 288 288 288 F F F 17 340-359 160 160 160 277 277 277 T T T 18 360-379 165 165 165 286 286 286 F F F 19 380-399 165 165 165 286 286 286 F F F 20 400-419 167 167 167 288 288 288 T T T 21 420-439 161 161 161 278 278 278 T T T 22 440-459 162 162 162 279 279 279 T T T 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 165 165 165 286 286 286 T T T 26 520-539 165 165 165 286 286 286 F F F 27 540-559 167 167 167 288 288 288 F F F 28 560-579 164 164 164 284 284 284 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 167, Max flush: 288 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 272 272 272 F F F 1 020-039 167 167 167 288 288 288 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 160 160 160 277 277 277 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 168 168 168 289 289 289 F F F 6 120-139 168 168 168 289 289 289 F F F 7 140-159 168 168 168 289 289 289 F F F 8 160-179 168 168 168 289 289 289 F F F 9 180-199 168 168 168 289 289 289 F F F 10 200-219 168 168 168 289 289 289 F F F 11 220-239 168 168 168 289 289 289 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 166 166 166 287 287 287 F F F 14 280-299 166 166 166 287 287 287 F F F 15 300-319 167 167 167 288 288 288 F F F 16 320-339 167 167 167 288 288 288 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 165 165 165 286 286 286 F F F 19 380-399 165 165 165 286 286 286 F F F 20 400-419 168 168 168 289 289 289 F F F 21 420-439 168 168 168 289 289 289 F F F 22 440-459 168 168 168 289 289 289 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 168 168 168 289 289 289 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 167 167 167 288 288 288 F F F 28 560-579 164 164 164 284 284 284 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 168, Max flush: 289 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.596169408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:40.639064692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.639417614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.671176747Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.671764375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.67179903Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.671812859Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.671827167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861} {"msg":"Crucible f3e9099f-0159-4a72-9482-fa7df2bf1d8b has session id: 4de15c5e-79c4-4a75-a283-6573602426a3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.672292222Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Upstairs opts: Upstairs UUID: f3e9099f-0159-4a72-9482-fa7df2bf1d8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.672338496Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Crucible stats registered with UUID: f3e9099f-0159-4a72-9482-fa7df2bf1d8b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.672365212Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.703893197Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861} {"msg":"f3e9099f-0159-4a72-9482-fa7df2bf1d8b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.703935071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.703964047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704010731Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704048126Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704083441Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704119376Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704154491Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704188936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"io task","client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704223202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"io task","client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704287663Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"io task","client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:65276)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704358293Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"io task","client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45725)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704395208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"io task","client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39421)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.704420455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"io task","client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705046838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705086932Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705118448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705153993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705201147Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705236382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705881713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.705932815Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706006855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706564738Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706599254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.70662633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706642157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706656466Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706670104Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706682302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.70669579Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[0]R flush_numbers[0..12]: [272, 288, 284, 277, 278, 289, 289, 289, 289, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706710028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[0]R generation[0..12]: [156, 167, 164, 160, 161, 168, 168, 168, 168, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706728336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706742834Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[1]R flush_numbers[0..12]: [272, 288, 284, 277, 278, 289, 289, 289, 289, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706756572Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[1]R generation[0..12]: [156, 167, 164, 160, 161, 168, 168, 168, 168, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.70677013Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706783928Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[2]R flush_numbers[0..12]: [272, 288, 284, 277, 278, 289, 289, 289, 289, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706798896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[2]R generation[0..12]: [156, 167, 164, 160, 161, 168, 168, 168, 168, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706823623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706856068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Max found gen is 169","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706879905Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Generation requested: 170 >= found:169","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706895183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706908331Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706921239Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"":"downstairs","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706934487Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706946335Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"0","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706962073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"1","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.706990729Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"client":"2","":"downstairs","session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.707005377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.707017666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"f3e9099f-0159-4a72-9482-fa7df2bf1d8b is now active with session: 4de15c5e-79c4-4a75-a283-6573602426a3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.707030594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.707042442Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861,"session_id":"4de15c5e-79c4-4a75-a283-6573602426a3"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:40.70705474Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 163 len 36864 data: 8 7 8 7 7 5 6 5 5 02/30 Write block 431 len 12288 data: 10 12 9 03/30 Read block 75 len 12288 04/30 Write block 405 len 12288 data: 16 16 14 05/30 Read block 530 len 16384 06/30 Read block 64 len 40960 07/30 Write block 141 len 20480 data: 13 10 12 11 13 08/30 Write block 172 len 8192 data: 5 6 09/30 Read block 90 len 20480 10/30 Write block 12 len 32768 data: 10 11 14 16 17 12 9 8 11/30 Read block 470 len 12288 12/30 Write block 120 len 36864 data: 17 17 17 14 15 18 18 18 18 13/30 Read block 582 len 16384 14/30 Read block 7 len 4096 15/30 Write block 306 len 40960 data: 15 13 10 7 9 9 10 9 10 10 16/30 Write block 282 len 8192 data: 13 14 17/30 Read block 166 len 32768 18/30 Read block 497 len 36864 19/30 Flush 20/30 Flush 21/30 Write block 565 len 36864 data: 12 17 15 16 17 14 13 12 12 22/30 Write block 509 len 40960 data: 8 7 6 9 10 16 17 16 17 16 23/30 Write block 551 len 16384 data: 8 8 13 15 24/30 Write block 63 len 20480 data: 15 17 19 19 23 25/30 Flush 26/30 Write block 133 len 16384 data: 17 15 11 13 27/30 Write block 247 len 40960 data: 24 22 21 17 15 13 9 9 9 10 28/30 Read block 423 len 28672 29/30 Read block 161 len 32768 30/30 Read block 516 len 16384 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:40.733860338Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1861} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 290 290 290 F F F 1 020-039 167 167 167 288 288 288 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 170 170 170 292 292 292 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 168 168 168 289 289 289 F F F 6 120-139 170 170 170 290 290 290 T T T 7 140-159 170 170 170 290 290 290 F F F 8 160-179 170 170 170 290 290 290 F F F 9 180-199 168 168 168 289 289 289 F F F 10 200-219 168 168 168 289 289 289 F F F 11 220-239 168 168 168 289 289 289 F F F 12 240-259 162 162 162 280 280 280 T T T 13 260-279 166 166 166 287 287 287 F F F 14 280-299 170 170 170 290 290 290 F F F 15 300-319 170 170 170 290 290 290 F F F 16 320-339 167 167 167 288 288 288 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 165 165 165 286 286 286 F F F 19 380-399 165 165 165 286 286 286 F F F 20 400-419 170 170 170 290 290 290 F F F 21 420-439 170 170 170 290 290 290 F F F 22 440-459 168 168 168 289 289 289 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 170 170 170 292 292 292 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 170 170 170 292 292 292 F F F 28 560-579 170 170 170 292 292 292 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 170, Max flush: 292 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 290 290 290 F F F 1 020-039 167 167 167 288 288 288 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 170 170 170 292 292 292 F F F 4 080-099 161 161 161 278 278 278 F F F 5 100-119 168 168 168 289 289 289 F F F 6 120-139 171 171 171 293 293 293 F F F 7 140-159 170 170 170 290 290 290 F F F 8 160-179 170 170 170 290 290 290 F F F 9 180-199 168 168 168 289 289 289 F F F 10 200-219 168 168 168 289 289 289 F F F 11 220-239 168 168 168 289 289 289 F F F 12 240-259 171 171 171 293 293 293 F F F 13 260-279 166 166 166 287 287 287 F F F 14 280-299 170 170 170 290 290 290 F F F 15 300-319 170 170 170 290 290 290 F F F 16 320-339 167 167 167 288 288 288 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 165 165 165 286 286 286 F F F 19 380-399 165 165 165 286 286 286 F F F 20 400-419 170 170 170 290 290 290 F F F 21 420-439 170 170 170 290 290 290 F F F 22 440-459 168 168 168 289 289 289 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 170 170 170 292 292 292 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 170 170 170 292 292 292 F F F 28 560-579 170 170 170 292 292 292 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 171, Max flush: 293 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.037366437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.082957979Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.083865154Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.083908698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.083930565Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.08396208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869} {"msg":"Crucible dab55664-9b03-45cc-bc05-006958159d91 has session id: a8f599b1-2dc5-4ac2-a269-44dd0eb577e7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.084419717Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Upstairs opts: Upstairs UUID: dab55664-9b03-45cc-bc05-006958159d91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.084480709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Crucible stats registered with UUID: dab55664-9b03-45cc-bc05-006958159d91","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.084511435Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116516794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869} {"msg":"dab55664-9b03-45cc-bc05-006958159d91 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116568236Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116589614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116620139Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116641586Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116668013Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.1166883Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116712657Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116736193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"io task","client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.11675821Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"io task","client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116781287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"io task","client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62051)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116803344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"io task","client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48205)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116826611Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"io task","client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60085)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.116851917Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"io task","client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.117534493Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.117562429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.117735655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.117767251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.117886554Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.117914501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.118262793Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.11842418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.118771262Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.118925281Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.118952037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.118966865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.11900276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119019258Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119032856Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119265744Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119288131Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[0]R flush_numbers[0..12]: [290, 288, 284, 292, 278, 289, 293, 290, 290, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119305509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[0]R generation[0..12]: [170, 167, 164, 170, 161, 168, 171, 170, 170, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119319656Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119333774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[1]R flush_numbers[0..12]: [290, 288, 284, 292, 278, 289, 293, 290, 290, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119347473Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[1]R generation[0..12]: [170, 167, 164, 170, 161, 168, 171, 170, 170, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.11936508Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119379788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[2]R flush_numbers[0..12]: [290, 288, 284, 292, 278, 289, 293, 290, 290, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119393536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[2]R generation[0..12]: [170, 167, 164, 170, 161, 168, 171, 170, 170, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119407124Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119421193Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119434161Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119447369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119460207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119473155Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"":"downstairs","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119486264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:56:45.119498292Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"0","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119516779Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"1","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119529987Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"client":"2","":"downstairs","session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119543166Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119555474Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"dab55664-9b03-45cc-bc05-006958159d91 is now active with session: a8f599b1-2dc5-4ac2-a269-44dd0eb577e7","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119570452Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.11958273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869,"session_id":"a8f599b1-2dc5-4ac2-a269-44dd0eb577e7"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:45.119598508Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 278 len 12288 data: 16 16 15 02/30 Write block 96 len 20480 data: 10 10 12 14 15 03/30 Read block 175 len 36864 04/30 Write block 64 len 32768 data: 18 20 20 24 19 21 21 21 05/30 Read block 25 len 16384 06/30 Read block 306 len 20480 07/30 Read block 504 len 16384 08/30 Read block 517 len 16384 09/30 Write block 146 len 28672 data: 14 14 13 13 11 11 13 10/30 Read block 411 len 28672 11/30 Flush 12/30 Write block 84 len 8192 data: 11 13 13/30 Write block 374 len 40960 data: 12 16 21 23 17 22 20 17 17 17 14/30 Read block 535 len 8192 15/30 Flush 16/30 Write block 29 len 12288 data: 11 12 12 17/30 Read block 541 len 8192 18/30 Write block 556 len 24576 data: 16 13 17 17 13 12 19/30 Flush 20/30 Write block 151 len 8192 data: 12 14 21/30 Read block 430 len 12288 22/30 Read block 544 len 16384 23/30 Read block 314 len 40960 24/30 Write block 562 len 20480 data: 9 10 11 13 18 25/30 Write block 91 len 16384 data: 12 11 10 9 26/30 Read block 460 len 24576 27/30 Write block 502 len 36864 data: 13 12 9 6 8 8 7 9 8 28/30 Write block 128 len 24576 data: 19 16 18 19 19 18 29/30 Read block 559 len 28672 30/30 Write block 310 len 40960 data: 10 10 11 10 11 11 10 13 12 15 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:45.146738969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1869} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 290 290 290 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 172 172 172 294 294 294 F F F 4 080-099 172 172 172 295 295 295 T T T 5 100-119 172 172 172 294 294 294 F F F 6 120-139 171 171 171 293 293 293 T T T 7 140-159 172 172 172 294 294 294 T T T 8 160-179 170 170 170 290 290 290 F F F 9 180-199 168 168 168 289 289 289 F F F 10 200-219 168 168 168 289 289 289 F F F 11 220-239 168 168 168 289 289 289 F F F 12 240-259 171 171 171 293 293 293 F F F 13 260-279 172 172 172 294 294 294 F F F 14 280-299 172 172 172 294 294 294 F F F 15 300-319 170 170 170 290 290 290 T T T 16 320-339 167 167 167 288 288 288 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 172 172 172 295 295 295 F F F 19 380-399 172 172 172 295 295 295 F F F 20 400-419 170 170 170 290 290 290 F F F 21 420-439 170 170 170 290 290 290 F F F 22 440-459 168 168 168 289 289 289 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 170 170 170 292 292 292 T T T 26 520-539 165 165 165 286 286 286 F F F 27 540-559 172 172 172 296 296 296 F F F 28 560-579 172 172 172 296 296 296 T T T 29 580-599 165 165 165 286 286 286 F F F Max gen: 172, Max flush: 296 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 290 290 290 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 172 172 172 294 294 294 F F F 4 080-099 173 173 173 297 297 297 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 297 297 297 F F F 7 140-159 173 173 173 297 297 297 F F F 8 160-179 170 170 170 290 290 290 F F F 9 180-199 168 168 168 289 289 289 F F F 10 200-219 168 168 168 289 289 289 F F F 11 220-239 168 168 168 289 289 289 F F F 12 240-259 171 171 171 293 293 293 F F F 13 260-279 172 172 172 294 294 294 F F F 14 280-299 172 172 172 294 294 294 F F F 15 300-319 173 173 173 297 297 297 F F F 16 320-339 167 167 167 288 288 288 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 172 172 172 295 295 295 F F F 19 380-399 172 172 172 295 295 295 F F F 20 400-419 170 170 170 290 290 290 F F F 21 420-439 170 170 170 290 290 290 F F F 22 440-459 168 168 168 289 289 289 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 172 172 172 296 296 296 F F F 28 560-579 173 173 173 297 297 297 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 173, Max flush: 297 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.483773587Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.529391776Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.530203414Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.530240258Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.530254547Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.530268655Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877} {"msg":"Crucible b4ada3d9-150a-4272-bb7f-f33aa71609cb has session id: c903b288-b32e-4744-8141-dd0dac5ee116","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.530753448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Upstairs opts: Upstairs UUID: b4ada3d9-150a-4272-bb7f-f33aa71609cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.530789653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Crucible stats registered with UUID: b4ada3d9-150a-4272-bb7f-f33aa71609cb","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.530805431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.561942179Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877} {"msg":"b4ada3d9-150a-4272-bb7f-f33aa71609cb active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.561979474Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.561994752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562017409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562032227Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562046355Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562060233Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.5620799Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562094878Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"io task","client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562114976Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"io task","client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562139672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"io task","client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_connection connected from Ok(127.0.0.1:63347)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562248727Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"io task","client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_connection connected from Ok(127.0.0.1:33903)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562285622Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"io task","client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_connection connected from Ok(127.0.0.1:47501)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.562313678Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"io task","client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.563185758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.563212654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.563229032Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.56324411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.563302942Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.563333317Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.563955411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.563984797Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564004885Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564608161Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564634998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564650265Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564662684Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564677232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.5646904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564702698Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564716026Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[0]R flush_numbers[0..12]: [290, 296, 284, 294, 297, 294, 297, 297, 290, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564730724Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[0]R generation[0..12]: [170, 172, 164, 172, 173, 172, 173, 173, 170, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564744762Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.56475939Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[1]R flush_numbers[0..12]: [290, 296, 284, 294, 297, 294, 297, 297, 290, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564773359Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[1]R generation[0..12]: [170, 172, 164, 172, 173, 172, 173, 173, 170, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564786977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564801024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[2]R flush_numbers[0..12]: [290, 296, 284, 294, 297, 294, 297, 297, 290, 289, 289, 289]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564818432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[2]R generation[0..12]: [170, 172, 164, 172, 173, 172, 173, 173, 170, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.56483266Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564846838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564859827Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564873165Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564885993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"All extents match",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564932017Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"":"downstairs","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564950024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564962632Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"0","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.56497615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"1","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.564989509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"client":"2","":"downstairs","session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.565002757Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.565014965Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"b4ada3d9-150a-4272-bb7f-f33aa71609cb is now active with session: c903b288-b32e-4744-8141-dd0dac5ee116","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.565027713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.565039612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877,"session_id":"c903b288-b32e-4744-8141-dd0dac5ee116"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:49.56505532Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 24576 data: 17 17 15 14 14 16 02/30 Read block 238 len 8192 03/30 Read block 92 len 36864 04/30 Write block 165 len 12288 data: 9 8 8 05/30 Write block 231 len 40960 data: 17 12 12 13 10 10 8 10 11 11 06/30 Write block 92 len 20480 data: 12 11 10 11 11 07/30 Read block 183 len 16384 08/30 Flush 09/30 Flush 10/30 Write block 255 len 20480 data: 10 11 12 11 10 11/30 Read block 125 len 36864 12/30 Flush 13/30 Write block 432 len 8192 data: 13 10 14/30 Read block 54 len 12288 15/30 Write block 308 len 32768 data: 11 8 11 11 12 11 12 12 16/30 Write block 182 len 20480 data: 16 17 13 11 12 17/30 Read block 359 len 20480 18/30 Read block 393 len 16384 19/30 Flush 20/30 Write block 78 len 24576 data: 14 13 12 9 9 11 21/30 Write block 86 len 16384 data: 12 13 15 14 22/30 Write block 536 len 8192 data: 16 12 23/30 Write block 314 len 20480 data: 13 13 11 14 13 24/30 Read block 64 len 4096 25/30 Read block 521 len 8192 26/30 Read block 335 len 8192 27/30 Write block 164 len 40960 data: 8 10 9 9 6 7 6 6 6 7 28/30 Write block 218 len 32768 data: 8 8 11 15 15 12 15 16 29/30 Read block 294 len 16384 30/30 Read block 175 len 28672 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 6 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:49.589643743Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1877} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 290 290 290 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 172 172 172 294 294 294 T T T 4 080-099 174 174 174 298 298 298 T T T 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 297 297 297 F F F 7 140-159 173 173 173 297 297 297 F F F 8 160-179 174 174 174 298 298 298 T T T 9 180-199 174 174 174 301 301 301 F F F 10 200-219 168 168 168 289 289 289 T T T 11 220-239 174 174 174 298 298 298 T T T 12 240-259 174 174 174 300 300 300 F F F 13 260-279 172 172 172 294 294 294 F F F 14 280-299 172 172 172 294 294 294 F F F 15 300-319 174 174 174 301 301 301 T T T 16 320-339 167 167 167 288 288 288 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 174 174 174 298 298 298 F F F 19 380-399 172 172 172 295 295 295 F F F 20 400-419 170 170 170 290 290 290 F F F 21 420-439 174 174 174 301 301 301 F F F 22 440-459 168 168 168 289 289 289 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 165 165 165 286 286 286 T T T 27 540-559 172 172 172 296 296 296 F F F 28 560-579 173 173 173 297 297 297 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 174, Max flush: 301 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 290 290 290 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 175 175 175 302 302 302 F F F 4 080-099 175 175 175 302 302 302 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 297 297 297 F F F 7 140-159 173 173 173 297 297 297 F F F 8 160-179 175 175 175 302 302 302 F F F 9 180-199 174 174 174 301 301 301 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 175 175 175 302 302 302 F F F 12 240-259 174 174 174 300 300 300 F F F 13 260-279 172 172 172 294 294 294 F F F 14 280-299 172 172 172 294 294 294 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 167 167 167 288 288 288 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 174 174 174 298 298 298 F F F 19 380-399 172 172 172 295 295 295 F F F 20 400-419 170 170 170 290 290 290 F F F 21 420-439 174 174 174 301 301 301 F F F 22 440-459 168 168 168 289 289 289 F F F 23 460-479 163 163 163 283 283 283 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 175 175 175 302 302 302 F F F 27 540-559 172 172 172 296 296 296 F F F 28 560-579 173 173 173 297 297 297 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 175, Max flush: 302 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.932313802Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.976380635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.977178705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.977233047Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.977270382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.977305637Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885} {"msg":"Crucible adea2ee2-f9fa-4a8b-881b-54a8f04c9dd0 has session id: 626a27dc-1c27-4873-955c-008b84b46c03","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.977849032Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Upstairs opts: Upstairs UUID: adea2ee2-f9fa-4a8b-881b-54a8f04c9dd0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.977888267Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Crucible stats registered with UUID: adea2ee2-f9fa-4a8b-881b-54a8f04c9dd0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:53.977907524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009374087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885} {"msg":"adea2ee2-f9fa-4a8b-881b-54a8f04c9dd0 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009420101Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009436239Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009458006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009473254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009487402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.00950139Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009598226Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009631332Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"io task","client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009667937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"io task","client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009698483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"io task","client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:43300)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009729118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"io task","client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:35282)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009773072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"io task","client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:61016)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.009804018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"io task","client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.010745018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.010773444Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.010929602Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.010968127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.010995053Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.011036498Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.011366012Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.011412296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.011676409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012037089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012065296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012081423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012094192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.01210749Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012120898Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012277616Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012296064Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[0]R flush_numbers[0..12]: [290, 296, 284, 302, 302, 294, 297, 297, 302, 301, 302, 302]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.01232285Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[0]R generation[0..12]: [170, 172, 164, 175, 175, 172, 173, 173, 175, 174, 175, 175]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012337368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012351676Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[1]R flush_numbers[0..12]: [290, 296, 284, 302, 302, 294, 297, 297, 302, 301, 302, 302]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012365324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[1]R generation[0..12]: [170, 172, 164, 175, 175, 172, 173, 173, 175, 174, 175, 175]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012379082Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.01239317Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[2]R flush_numbers[0..12]: [290, 296, 284, 302, 302, 294, 297, 297, 302, 301, 302, 302]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012406938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[2]R generation[0..12]: [170, 172, 164, 175, 175, 172, 173, 173, 175, 174, 175, 175]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012423686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012438064Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012450922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.01246403Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012476989Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012490627Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"":"downstairs","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012503795Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:56:54.012515943Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"0","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012532921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"1","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012546049Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"client":"2","":"downstairs","session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012559477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012571906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"adea2ee2-f9fa-4a8b-881b-54a8f04c9dd0 is now active with session: 626a27dc-1c27-4873-955c-008b84b46c03","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012585074Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.012597322Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885,"session_id":"626a27dc-1c27-4873-955c-008b84b46c03"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:54.01260967Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 397 len 20480 data: 11 11 11 11 12 02/30 Read block 447 len 16384 03/30 Write block 8 len 20480 data: 15 13 15 10 11 04/30 Read block 247 len 12288 05/30 Write block 447 len 32768 data: 9 12 14 10 9 10 13 12 06/30 Read block 152 len 16384 07/30 Read block 220 len 16384 08/30 Read block 181 len 12288 09/30 Write block 197 len 12288 data: 15 13 9 10/30 Read block 577 len 20480 11/30 Write block 573 len 4096 data: 13 12/30 Write block 84 len 24576 data: 12 14 13 14 16 15 13/30 Read block 116 len 12288 14/30 Read block 523 len 12288 15/30 Read block 283 len 40960 16/30 Write block 460 len 36864 data: 18 17 14 16 16 18 14 12 12 17/30 Read block 113 len 32768 18/30 Read block 333 len 40960 19/30 Read block 566 len 28672 20/30 Write block 251 len 36864 data: 16 14 10 10 11 12 13 12 11 21/30 Write block 233 len 20480 data: 13 14 11 11 9 22/30 Read block 204 len 16384 23/30 Read block 376 len 24576 24/30 Write block 338 len 12288 data: 10 8 10 25/30 Read block 269 len 8192 26/30 Read block 446 len 28672 27/30 Write block 384 len 16384 data: 14 8 8 11 28/30 Read block 144 len 28672 29/30 Read block 251 len 8192 30/30 Write block 451 len 32768 data: 10 11 14 13 13 15 17 18 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 5 Done Done Done false 2 Acked 1001 Read 4 Done Done Done false 3 Acked 1002 Write 5 Done Done Done false 4 Acked 1003 Read 3 Done Done Done false 5 Acked 1004 Write 8 Done Done Done false 6 Acked 1005 Read 4 Done Done Done false 7 Acked 1006 Read 4 Done Done Done false 8 Acked 1007 Read 3 Done Done Done false 9 Acked 1008 Write 3 Done Done Done false 10 Acked 1009 Read 5 Done Done Done false 11 Acked 1010 Write 1 Done Done Done false 12 Acked 1011 Write 6 Done Done Done false 13 Acked 1012 Read 3 Done Done Done false 14 Acked 1013 Read 3 Done Done Done false 15 Acked 1014 Read 10 Done Done Done false 16 Acked 1015 Write 9 Done Done Done false 17 Acked 1016 Read 8 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Read 7 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:54.03404382Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1885} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 290 290 290 T T T 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 175 175 175 302 302 302 F F F 4 080-099 175 175 175 302 302 302 T T T 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 297 297 297 F F F 7 140-159 173 173 173 297 297 297 F F F 8 160-179 175 175 175 302 302 302 F F F 9 180-199 174 174 174 301 301 301 T T T 10 200-219 175 175 175 302 302 302 F F F 11 220-239 175 175 175 302 302 302 T T T 12 240-259 174 174 174 300 300 300 T T T 13 260-279 172 172 172 294 294 294 F F F 14 280-299 172 172 172 294 294 294 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 167 167 167 288 288 288 T T T 17 340-359 168 168 168 289 289 289 T T T 18 360-379 174 174 174 298 298 298 F F F 19 380-399 172 172 172 295 295 295 T T T 20 400-419 170 170 170 290 290 290 T T T 21 420-439 174 174 174 301 301 301 F F F 22 440-459 168 168 168 289 289 289 T T T 23 460-479 163 163 163 283 283 283 T T T 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 175 175 175 302 302 302 F F F 27 540-559 172 172 172 296 296 296 F F F 28 560-579 173 173 173 297 297 297 T T T 29 580-599 165 165 165 286 286 286 F F F Max gen: 175, Max flush: 302 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 303 303 303 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 175 175 175 302 302 302 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 297 297 297 F F F 7 140-159 173 173 173 297 297 297 F F F 8 160-179 175 175 175 302 302 302 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 176 176 176 303 303 303 F F F 12 240-259 176 176 176 303 303 303 F F F 13 260-279 172 172 172 294 294 294 F F F 14 280-299 172 172 172 294 294 294 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 176 176 176 303 303 303 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 174 174 174 298 298 298 F F F 19 380-399 176 176 176 303 303 303 F F F 20 400-419 176 176 176 303 303 303 F F F 21 420-439 174 174 174 301 301 301 F F F 22 440-459 176 176 176 303 303 303 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 175 175 175 302 302 302 F F F 27 540-559 172 172 172 296 296 296 F F F 28 560-579 176 176 176 303 303 303 F F F 29 580-599 165 165 165 286 286 286 F F F Max gen: 176, Max flush: 303 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.457694443Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.503520093Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.504399042Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.504436937Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.504451255Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.504465643Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893} {"msg":"Crucible 12a4bf72-8467-49b1-937a-447bbdfcfb48 has session id: 6eaa50f8-93d7-4197-909d-e969c6e4c21c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.504957225Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Upstairs opts: Upstairs UUID: 12a4bf72-8467-49b1-937a-447bbdfcfb48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.505012057Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Crucible stats registered with UUID: 12a4bf72-8467-49b1-937a-447bbdfcfb48","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.505029635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537264712Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893} {"msg":"12a4bf72-8467-49b1-937a-447bbdfcfb48 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537313445Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537331253Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537359869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537377577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537408512Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537440268Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537474223Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537506948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"io task","client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537545713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"io task","client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537581458Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58819)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537618373Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"io task","client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59858)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537650079Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34888)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.537900624Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"io task","client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.538505321Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.538531907Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.538546385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.538560113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.538573391Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.538586649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.539274175Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.539313239Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.539350544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.539858294Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.53988478Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.539902068Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.539992165Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540026451Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540045448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540060896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540077224Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[0]R flush_numbers[0..12]: [303, 296, 284, 302, 303, 294, 297, 297, 302, 303, 302, 303]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540145034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[0]R generation[0..12]: [176, 172, 164, 175, 176, 172, 173, 173, 175, 176, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540165281Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540180109Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[1]R flush_numbers[0..12]: [303, 296, 284, 302, 303, 294, 297, 297, 302, 303, 302, 303]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540194117Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[1]R generation[0..12]: [176, 172, 164, 175, 176, 172, 173, 173, 175, 176, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540207735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540221653Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[2]R flush_numbers[0..12]: [303, 296, 284, 302, 303, 294, 297, 297, 302, 303, 302, 303]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540239401Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[2]R generation[0..12]: [176, 172, 164, 175, 176, 172, 173, 173, 175, 176, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540253659Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540267617Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Max found gen is 177","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540280705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Generation requested: 178 >= found:177","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540293774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540306732Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.54031984Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"":"downstairs","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540333118Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540345167Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"0","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:56:58.540358505Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"1","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540437954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"client":"2","":"downstairs","session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540455171Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.54046803Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"12a4bf72-8467-49b1-937a-447bbdfcfb48 is now active with session: 6eaa50f8-93d7-4197-909d-e969c6e4c21c","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540480938Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540492926Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893,"session_id":"6eaa50f8-93d7-4197-909d-e969c6e4c21c"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:56:58.540507904Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 28672 data: 15 13 13 10 9 9 11 02/30 Read block 275 len 20480 03/30 Read block 545 len 20480 04/30 Read block 560 len 4096 05/30 Read block 113 len 28672 06/30 Write block 384 len 12288 data: 15 9 9 07/30 Read block 439 len 32768 08/30 Read block 91 len 32768 09/30 Read block 119 len 36864 10/30 Write block 68 len 28672 data: 20 22 22 22 20 20 15 11/30 Flush 12/30 Flush 13/30 Write block 239 len 28672 data: 12 12 8 9 12 14 15 14/30 Read block 147 len 24576 15/30 Flush 16/30 Write block 238 len 8192 data: 11 13 17/30 Read block 493 len 32768 18/30 Write block 469 len 28672 data: 13 14 15 14 10 12 10 19/30 Read block 234 len 40960 20/30 Write block 277 len 16384 data: 16 17 17 16 21/30 Flush 22/30 Write block 432 len 12288 data: 14 11 9 23/30 Read block 568 len 16384 24/30 Read block 142 len 20480 25/30 Write block 373 len 12288 data: 11 13 17 26/30 Write block 13 len 20480 data: 12 15 17 18 13 27/30 Write block 589 len 24576 data: 9 11 11 11 9 9 28/30 Write block 137 len 8192 data: 12 14 29/30 Write block 381 len 40960 data: 18 18 18 16 10 10 12 14 13 16 30/30 Read block 66 len 20480 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 5 Sent Done Done false 27 Acked 1026 Write 6 Sent Done Done false 28 Acked 1027 Write 2 Sent Done Done false 29 Acked 1028 Write 10 Sent Done Done false 30 Acked 1029 Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 5 1 0 6 Done 4 8 9 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:56:58.567256819Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1893} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 303 303 303 T T T 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 173 173 173 297 297 297 T T T 7 140-159 178 178 178 304 304 304 F F F 8 160-179 178 178 178 304 304 304 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 178 178 178 307 307 307 F F F 12 240-259 178 178 178 306 306 306 F F F 13 260-279 178 178 178 307 307 307 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 176 176 176 303 303 303 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 174 174 174 298 298 298 T T T 19 380-399 178 178 178 304 304 304 T T T 20 400-419 176 176 176 303 303 303 F F F 21 420-439 174 174 174 301 301 301 T T T 22 440-459 176 176 176 303 303 303 F F F 23 460-479 178 178 178 307 307 307 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 175 175 175 302 302 302 F F F 27 540-559 172 172 172 296 296 296 F F F 28 560-579 176 176 176 303 303 303 F F F 29 580-599 165 165 165 286 286 286 T T T Max gen: 178, Max flush: 307 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 308 308 308 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 179 179 179 308 308 308 F F F 7 140-159 178 178 178 304 304 304 F F F 8 160-179 178 178 178 304 304 304 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 178 178 178 307 307 307 F F F 12 240-259 178 178 178 306 306 306 F F F 13 260-279 178 178 178 307 307 307 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 176 176 176 303 303 303 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 179 179 179 308 308 308 F F F 19 380-399 179 179 179 308 308 308 F F F 20 400-419 176 176 176 303 303 303 F F F 21 420-439 179 179 179 308 308 308 F F F 22 440-459 176 176 176 303 303 303 F F F 23 460-479 178 178 178 307 307 307 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 175 175 175 302 302 302 F F F 27 540-559 172 172 172 296 296 296 F F F 28 560-579 176 176 176 303 303 303 F F F 29 580-599 179 179 179 308 308 308 F F F Max gen: 179, Max flush: 308 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.903820872Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.948048393Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.948748156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.948785171Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.948804388Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.948819576Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901} {"msg":"Crucible 503fb789-6373-438d-a462-b9db48b4f377 has session id: fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.949387737Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Upstairs opts: Upstairs UUID: 503fb789-6373-438d-a462-b9db48b4f377, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.949430842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Crucible stats registered with UUID: 503fb789-6373-438d-a462-b9db48b4f377","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.949454778Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980547863Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901} {"msg":"503fb789-6373-438d-a462-b9db48b4f377 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980583188Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980598546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980619323Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980633441Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.98064713Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980661157Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980674686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980688024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980701852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.980720909Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49435)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.981092728Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50426)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.981192924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35544)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.981252516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.981992864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.98202174Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.982037278Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.982051296Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.982064764Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.982121696Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.982702076Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.982730832Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.982785814Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983349046Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983387241Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983417717Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983433195Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983447333Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983460871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983473039Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983486107Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[0]R flush_numbers[0..12]: [308, 296, 284, 304, 303, 294, 308, 304, 304, 303, 302, 307]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983499656Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[0]R generation[0..12]: [179, 172, 164, 178, 176, 172, 179, 178, 178, 176, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983513543Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983527592Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[1]R flush_numbers[0..12]: [308, 296, 284, 304, 303, 294, 308, 304, 304, 303, 302, 307]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.98354126Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[1]R generation[0..12]: [179, 172, 164, 178, 176, 172, 179, 178, 178, 176, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983554978Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983568776Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[2]R flush_numbers[0..12]: [308, 296, 284, 304, 303, 294, 308, 304, 304, 303, 302, 307]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983582524Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[2]R generation[0..12]: [179, 172, 164, 178, 176, 172, 179, 178, 178, 176, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983596202Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983621689Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Max found gen is 180","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2024-09-27T03:57:02.983704197Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983735803Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983751271Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983764889Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"":"downstairs","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983778207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983790505Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"0","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983804094Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"1","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983817581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"client":"2","":"downstairs","session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.98383109Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983843608Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"503fb789-6373-438d-a462-b9db48b4f377 is now active with session: fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983856646Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983868765Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901,"session_id":"fef3b4fb-0b2d-4a1f-b4e6-8adab94b908b"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:02.983881023Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 429 len 8192 data: 9 9 02/30 Read block 589 len 4096 03/30 Write block 341 len 28672 data: 12 12 9 11 9 14 15 04/30 Write block 256 len 36864 data: 13 14 13 12 14 16 12 11 11 05/30 Read block 564 len 4096 06/30 Read block 405 len 24576 07/30 Read block 338 len 28672 08/30 Write block 305 len 36864 data: 17 16 14 12 9 12 12 13 12 09/30 Write block 558 len 40960 data: 18 18 14 13 10 11 12 14 19 16 10/30 Flush 11/30 Write block 450 len 36864 data: 11 11 12 15 14 14 16 18 19 12/30 Flush 13/30 Write block 568 len 24576 data: 17 18 15 14 13 14 14/30 Write block 318 len 8192 data: 14 16 15/30 Write block 121 len 28672 data: 18 18 15 16 19 19 19 16/30 Write block 430 len 12288 data: 10 11 15 17/30 Write block 434 len 4096 data: 10 18/30 Read block 243 len 40960 19/30 Write block 518 len 20480 data: 17 14 13 12 11 20/30 Write block 50 len 4096 data: 16 21/30 Write block 302 len 32768 data: 14 12 16 18 17 15 13 10 22/30 Read block 91 len 4096 23/30 Write block 270 len 32768 data: 10 11 13 12 11 16 16 17 24/30 Write block 467 len 40960 data: 13 13 14 15 16 15 11 13 11 12 25/30 Write block 172 len 20480 data: 7 8 9 9 10 26/30 Write block 357 len 36864 data: 14 12 11 15 17 18 18 16 15 27/30 Read block 299 len 28672 28/30 Read block 371 len 40960 29/30 Write block 227 len 24576 data: 20 21 19 19 18 13 30/30 Write block 557 len 28672 data: 14 19 19 15 14 11 12 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 6 Done Done Done false 14 Acked 1013 Write 2 Done Done Done false 15 Acked 1014 Write 7 Done Done Done false 16 Acked 1015 Write 3 Done Done Done false 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Write 6 Sent Sent Sent false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:03.005323421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1901} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 308 308 308 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 164 164 164 284 284 284 T T T 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 179 179 179 308 308 308 T T T 7 140-159 178 178 178 304 304 304 F F F 8 160-179 178 178 178 304 304 304 T T T 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 178 178 178 307 307 307 T T T 12 240-259 180 180 180 309 309 309 F F F 13 260-279 180 180 180 309 309 309 T T T 14 280-299 178 178 178 307 307 307 F F F 15 300-319 180 180 180 309 309 309 T T T 16 320-339 176 176 176 303 303 303 F F F 17 340-359 180 180 180 309 309 309 T T T 18 360-379 179 179 179 308 308 308 T T T 19 380-399 179 179 179 308 308 308 F F F 20 400-419 176 176 176 303 303 303 F F F 21 420-439 180 180 180 309 309 309 T T T 22 440-459 180 180 180 310 310 310 F F F 23 460-479 178 178 178 307 307 307 T T T 24 480-499 159 159 159 275 275 275 F F F 25 500-519 173 173 173 297 297 297 T T T 26 520-539 175 175 175 302 302 302 T T T 27 540-559 180 180 180 309 309 309 T T T 28 560-579 180 180 180 309 309 309 T T T 29 580-599 179 179 179 308 308 308 F F F Max gen: 180, Max flush: 310 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 308 308 308 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 181 181 181 311 311 311 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 172 172 172 294 294 294 F F F 6 120-139 181 181 181 311 311 311 F F F 7 140-159 178 178 178 304 304 304 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 181 181 181 311 311 311 F F F 12 240-259 180 180 180 309 309 309 F F F 13 260-279 181 181 181 311 311 311 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 181 181 181 311 311 311 F F F 16 320-339 176 176 176 303 303 303 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 181 181 181 311 311 311 F F F 19 380-399 179 179 179 308 308 308 F F F 20 400-419 176 176 176 303 303 303 F F F 21 420-439 181 181 181 311 311 311 F F F 22 440-459 180 180 180 310 310 310 F F F 23 460-479 181 181 181 311 311 311 F F F 24 480-499 159 159 159 275 275 275 F F F 25 500-519 181 181 181 311 311 311 F F F 26 520-539 181 181 181 311 311 311 F F F 27 540-559 181 181 181 311 311 311 F F F 28 560-579 181 181 181 311 311 311 F F F 29 580-599 179 179 179 308 308 308 F F F Max gen: 181, Max flush: 311 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.444035484Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.489582133Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.493010989Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.493057513Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.493081479Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.493100717Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909} {"msg":"Crucible fa745b9a-0f54-4a83-a010-aa5538ab055d has session id: ec70df74-0fd1-4d62-9b18-c5d57fadaa5b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.493542196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Upstairs opts: Upstairs UUID: fa745b9a-0f54-4a83-a010-aa5538ab055d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.493590179Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Crucible stats registered with UUID: fa745b9a-0f54-4a83-a010-aa5538ab055d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.493619495Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525701033Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909} {"msg":"fa745b9a-0f54-4a83-a010-aa5538ab055d active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525746087Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525765564Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525794791Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525812628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525829766Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525846473Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.52586997Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525898846Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525933921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525962827Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64720)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.525992473Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35858)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.526022809Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33290)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.526054425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.526794482Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.526833647Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.526851954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.526866532Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.527086842Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.527114098Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.52761045Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.527644465Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.527759869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528312473Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528343888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528358836Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528371404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528384883Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528398031Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528409969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528423227Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[0]R flush_numbers[0..12]: [308, 296, 311, 304, 303, 294, 311, 304, 311, 303, 302, 311]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528436895Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[0]R generation[0..12]: [179, 172, 181, 178, 176, 172, 181, 178, 181, 176, 175, 181]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528450824Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528464672Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[1]R flush_numbers[0..12]: [308, 296, 311, 304, 303, 294, 311, 304, 311, 303, 302, 311]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.52847857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[1]R generation[0..12]: [179, 172, 181, 178, 176, 172, 181, 178, 181, 176, 175, 181]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528492368Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528506266Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[2]R flush_numbers[0..12]: [308, 296, 311, 304, 303, 294, 311, 304, 311, 303, 302, 311]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528519964Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[2]R generation[0..12]: [179, 172, 181, 178, 176, 172, 181, 178, 181, 176, 175, 181]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528533752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.52855062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528564028Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528577256Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528590214Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528603272Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"":"downstairs","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.52862387Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528636408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"0","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528649556Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"1","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528662654Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"client":"2","":"downstairs","session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528675973Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528688271Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"fa745b9a-0f54-4a83-a010-aa5538ab055d is now active with session: ec70df74-0fd1-4d62-9b18-c5d57fadaa5b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528701139Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528713187Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909,"session_id":"ec70df74-0fd1-4d62-9b18-c5d57fadaa5b"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:07.528725306Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 4096 data: 16 02/30 Flush 03/30 Write block 9 len 32768 data: 14 16 11 12 13 16 18 19 04/30 Read block 276 len 24576 05/30 Read block 525 len 36864 06/30 Write block 494 len 32768 data: 8 8 11 12 11 11 11 11 07/30 Read block 236 len 16384 08/30 Read block 188 len 16384 09/30 Read block 144 len 8192 10/30 Write block 589 len 4096 data: 10 11/30 Read block 572 len 4096 12/30 Write block 587 len 8192 data: 9 10 13/30 Read block 474 len 24576 14/30 Write block 528 len 24576 data: 14 13 16 18 17 17 15/30 Read block 50 len 20480 16/30 Write block 104 len 12288 data: 8 9 10 17/30 Write block 517 len 40960 data: 18 18 15 14 13 12 12 13 14 14 18/30 Flush 19/30 Write block 368 len 8192 data: 17 16 20/30 Read block 408 len 16384 21/30 Read block 68 len 16384 22/30 Write block 592 len 24576 data: 12 10 10 9 5 6 23/30 Write block 401 len 4096 data: 13 24/30 Read block 4 len 40960 25/30 Read block 149 len 4096 26/30 Read block 560 len 20480 27/30 Read block 366 len 28672 28/30 Write block 557 len 36864 data: 15 20 20 16 15 12 13 13 15 29/30 Read block 15 len 12288 30/30 Read block 358 len 4096 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:07.552736789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1909} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 313 313 313 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 181 181 181 311 311 311 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 182 182 182 313 313 313 F F F 6 120-139 181 181 181 311 311 311 F F F 7 140-159 178 178 178 304 304 304 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 181 181 181 311 311 311 F F F 12 240-259 180 180 180 309 309 309 F F F 13 260-279 181 181 181 311 311 311 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 181 181 181 311 311 311 F F F 16 320-339 176 176 176 303 303 303 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 181 181 181 311 311 311 T T T 19 380-399 179 179 179 308 308 308 F F F 20 400-419 176 176 176 303 303 303 T T T 21 420-439 181 181 181 311 311 311 F F F 22 440-459 182 182 182 312 312 312 F F F 23 460-479 181 181 181 311 311 311 F F F 24 480-499 182 182 182 313 313 313 F F F 25 500-519 182 182 182 313 313 313 F F F 26 520-539 182 182 182 313 313 313 F F F 27 540-559 181 181 181 311 311 311 T T T 28 560-579 181 181 181 311 311 311 T T T 29 580-599 182 182 182 313 313 313 T T T Max gen: 182, Max flush: 313 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 313 313 313 F F F 1 020-039 172 172 172 296 296 296 F F F 2 040-059 181 181 181 311 311 311 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 182 182 182 313 313 313 F F F 6 120-139 181 181 181 311 311 311 F F F 7 140-159 178 178 178 304 304 304 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 F F F 11 220-239 181 181 181 311 311 311 F F F 12 240-259 180 180 180 309 309 309 F F F 13 260-279 181 181 181 311 311 311 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 181 181 181 311 311 311 F F F 16 320-339 176 176 176 303 303 303 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 183 183 183 314 314 314 F F F 19 380-399 179 179 179 308 308 308 F F F 20 400-419 183 183 183 314 314 314 F F F 21 420-439 181 181 181 311 311 311 F F F 22 440-459 182 182 182 312 312 312 F F F 23 460-479 181 181 181 311 311 311 F F F 24 480-499 182 182 182 313 313 313 F F F 25 500-519 182 182 182 313 313 313 F F F 26 520-539 182 182 182 313 313 313 F F F 27 540-559 183 183 183 314 314 314 F F F 28 560-579 183 183 183 314 314 314 F F F 29 580-599 183 183 183 314 314 314 F F F Max gen: 183, Max flush: 314 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.88337683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.929093525Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.93007527Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.930112534Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.930126882Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.93014095Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917} {"msg":"Crucible 553043cb-6428-4c61-9d0b-b730485337e3 has session id: 174e5595-6761-4d29-b1e1-41e48913dd63","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.93058054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Upstairs opts: Upstairs UUID: 553043cb-6428-4c61-9d0b-b730485337e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.930620024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Crucible stats registered with UUID: 553043cb-6428-4c61-9d0b-b730485337e3","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.930644641Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962332284Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917} {"msg":"553043cb-6428-4c61-9d0b-b730485337e3 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962385666Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962404994Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962438949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962463136Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962494821Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962527457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962556743Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962583379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962611185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962640101Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_connection connected from Ok(127.0.0.1:64898)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962676886Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_connection connected from Ok(127.0.0.1:36878)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962710151Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_connection connected from Ok(127.0.0.1:47347)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.962743477Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.963652411Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.963679577Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.963694326Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.963707924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.963721312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.96373471Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.96431564Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.964356024Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.964425454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.964942683Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.964983008Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965009714Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.9650371Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965053448Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965067816Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965080054Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965093302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[0]R flush_numbers[0..12]: [313, 296, 311, 304, 303, 313, 311, 304, 311, 303, 302, 311]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.96510723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[0]R generation[0..12]: [182, 172, 181, 178, 176, 182, 181, 178, 181, 176, 175, 181]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965121548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965135567Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[1]R flush_numbers[0..12]: [313, 296, 311, 304, 303, 313, 311, 304, 311, 303, 302, 311]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965149454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[1]R generation[0..12]: [182, 172, 181, 178, 176, 182, 181, 178, 181, 176, 175, 181]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965163583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.96517789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[2]R flush_numbers[0..12]: [313, 296, 311, 304, 303, 313, 311, 304, 311, 303, 302, 311]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965192648Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[2]R generation[0..12]: [182, 172, 181, 178, 176, 182, 181, 178, 181, 176, 175, 181]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965206377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965220065Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965235263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965248631Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965261459Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965274257Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"":"downstairs","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965287395Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:57:11.965299614Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"0","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965323001Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"1","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965349657Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"client":"2","":"downstairs","session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965376403Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.96540201Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"553043cb-6428-4c61-9d0b-b730485337e3 is now active with session: 174e5595-6761-4d29-b1e1-41e48913dd63","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965428966Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965453323Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917,"session_id":"174e5595-6761-4d29-b1e1-41e48913dd63"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:11.965479399Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 421 len 16384 data: 10 9 6 9 02/30 Flush 03/30 Write block 553 len 12288 data: 14 16 15 04/30 Read block 319 len 20480 05/30 Write block 587 len 8192 data: 10 11 06/30 Flush 07/30 Write block 252 len 24576 data: 15 11 11 12 14 15 08/30 Read block 213 len 32768 09/30 Write block 253 len 40960 data: 12 12 13 15 16 14 13 15 17 13 10/30 Read block 521 len 28672 11/30 Write block 376 len 16384 data: 22 24 18 23 12/30 Write block 386 len 24576 data: 11 13 15 14 17 12 13/30 Read block 359 len 24576 14/30 Read block 67 len 24576 15/30 Write block 141 len 28672 data: 14 11 13 12 14 15 15 16/30 Read block 404 len 20480 17/30 Flush 18/30 Write block 12 len 16384 data: 13 14 17 19 19/30 Write block 520 len 20480 data: 15 14 13 13 14 20/30 Write block 278 len 8192 data: 18 18 21/30 Write block 458 len 28672 data: 20 20 19 18 15 17 17 22/30 Read block 471 len 40960 23/30 Write block 17 len 40960 data: 14 10 9 11 11 14 12 11 7 8 24/30 Read block 80 len 12288 25/30 Write block 328 len 12288 data: 14 14 13 26/30 Write block 131 len 8192 data: 20 20 27/30 Read block 465 len 28672 28/30 Write block 215 len 24576 data: 8 8 9 9 9 12 29/30 Read block 235 len 40960 30/30 Read block 587 len 20480 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:11.992643497Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1917} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 313 313 313 T T T 1 020-039 172 172 172 296 296 296 T T T 2 040-059 181 181 181 311 311 311 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 182 182 182 313 313 313 F F F 6 120-139 181 181 181 311 311 311 T T T 7 140-159 184 184 184 317 317 317 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 175 175 175 302 302 302 T T T 11 220-239 181 181 181 311 311 311 T T T 12 240-259 184 184 184 317 317 317 F F F 13 260-279 184 184 184 317 317 317 T T T 14 280-299 178 178 178 307 307 307 F F F 15 300-319 181 181 181 311 311 311 F F F 16 320-339 176 176 176 303 303 303 T T T 17 340-359 181 181 181 311 311 311 F F F 18 360-379 184 184 184 317 317 317 F F F 19 380-399 184 184 184 317 317 317 F F F 20 400-419 183 183 183 314 314 314 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 182 182 182 312 312 312 T T T 23 460-479 181 181 181 311 311 311 T T T 24 480-499 182 182 182 313 313 313 F F F 25 500-519 182 182 182 313 313 313 F F F 26 520-539 182 182 182 313 313 313 T T T 27 540-559 184 184 184 316 316 316 F F F 28 560-579 183 183 183 314 314 314 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 184, Max flush: 317 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 318 318 318 F F F 1 020-039 185 185 185 318 318 318 F F F 2 040-059 181 181 181 311 311 311 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 182 182 182 313 313 313 F F F 6 120-139 185 185 185 318 318 318 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 185 185 185 318 318 318 F F F 11 220-239 185 185 185 318 318 318 F F F 12 240-259 184 184 184 317 317 317 F F F 13 260-279 185 185 185 318 318 318 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 181 181 181 311 311 311 F F F 16 320-339 185 185 185 318 318 318 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 184 184 184 317 317 317 F F F 19 380-399 184 184 184 317 317 317 F F F 20 400-419 183 183 183 314 314 314 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 185 185 185 318 318 318 F F F 23 460-479 185 185 185 318 318 318 F F F 24 480-499 182 182 182 313 313 313 F F F 25 500-519 182 182 182 313 313 313 F F F 26 520-539 185 185 185 318 318 318 F F F 27 540-559 184 184 184 316 316 316 F F F 28 560-579 183 183 183 314 314 314 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 185, Max flush: 318 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.384093408Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.429304993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.430074267Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.430116071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.430133379Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.430152396Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925} {"msg":"Crucible f56f42e1-999e-4eb6-b154-9052823bc7e4 has session id: 2358e25a-3298-4b43-be3b-cb128b9a99a0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.430637589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Upstairs opts: Upstairs UUID: f56f42e1-999e-4eb6-b154-9052823bc7e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.43069821Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Crucible stats registered with UUID: f56f42e1-999e-4eb6-b154-9052823bc7e4","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.430726037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462011695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925} {"msg":"f56f42e1-999e-4eb6-b154-9052823bc7e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.4620485Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462065708Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462095113Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462112801Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462137238Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462169323Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462202789Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462236094Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462269289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462302385Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50781)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462333831Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58403)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462667085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64375)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.462732955Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.463141669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.463172625Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.463310745Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.463332402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.463531605Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.46356856Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.463790829Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.463975354Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464117234Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464359251Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464384207Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464399015Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464458177Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464477774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464495032Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464697644Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464716931Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 311, 304, 303, 313, 318, 317, 311, 303, 318, 318]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464743108Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[0]R generation[0..12]: [185, 185, 181, 178, 176, 182, 185, 184, 181, 176, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464757446Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464772314Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 311, 304, 303, 313, 318, 317, 311, 303, 318, 318]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464786312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[1]R generation[0..12]: [185, 185, 181, 178, 176, 182, 185, 184, 181, 176, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.46480088Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464814968Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 311, 304, 303, 313, 318, 317, 311, 303, 318, 318]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464828426Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[2]R generation[0..12]: [185, 185, 181, 178, 176, 182, 185, 184, 181, 176, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464842254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464856072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.46486908Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464882208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464899276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464913274Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"":"downstairs","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464926782Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2024-09-27T03:57:16.46494517Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"0","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464959018Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"1","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464972636Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"client":"2","":"downstairs","session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464986514Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.464999122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"f56f42e1-999e-4eb6-b154-9052823bc7e4 is now active with session: 2358e25a-3298-4b43-be3b-cb128b9a99a0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.465011921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.465023869Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925,"session_id":"2358e25a-3298-4b43-be3b-cb128b9a99a0"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:16.465035957Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 478 len 8192 data: 12 17 02/30 Write block 529 len 40960 data: 14 17 19 18 18 16 14 17 13 12 03/30 Read block 294 len 36864 04/30 Write block 503 len 40960 data: 13 10 7 9 9 8 10 9 7 10 05/30 Write block 108 len 20480 data: 12 10 11 8 8 06/30 Write block 480 len 28672 data: 17 14 15 15 14 14 13 07/30 Read block 172 len 12288 08/30 Write block 317 len 16384 data: 15 15 17 12 09/30 Read block 157 len 40960 10/30 Write block 199 len 8192 data: 10 7 11/30 Write block 67 len 4096 data: 25 12/30 Read block 548 len 28672 13/30 Flush 14/30 Write block 11 len 32768 data: 12 14 15 18 20 20 15 11 15/30 Read block 378 len 36864 16/30 Read block 360 len 40960 17/30 Write block 330 len 20480 data: 14 12 13 15 15 18/30 Read block 421 len 36864 19/30 Flush 20/30 Write block 215 len 16384 data: 9 9 10 10 21/30 Flush 22/30 Write block 480 len 28672 data: 18 15 16 16 15 15 14 23/30 Read block 253 len 12288 24/30 Read block 530 len 28672 25/30 Read block 207 len 24576 26/30 Write block 76 len 16384 data: 15 16 15 14 27/30 Read block 240 len 16384 28/30 Read block 175 len 36864 29/30 Write block 529 len 12288 data: 15 18 20 30/30 Write block 26 len 32768 data: 9 10 12 12 13 13 11 12 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 3 Done Done Sent false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:16.490148538Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1925} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 185 185 185 318 318 318 T T T 2 040-059 181 181 181 311 311 311 F F F 3 060-079 186 186 186 319 319 319 T T T 4 080-099 176 176 176 303 303 303 F F F 5 100-119 186 186 186 319 319 319 F F F 6 120-139 185 185 185 318 318 318 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 186 186 186 319 319 319 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 185 185 185 318 318 318 F F F 12 240-259 184 184 184 317 317 317 F F F 13 260-279 185 185 185 318 318 318 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 186 186 186 319 319 319 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 184 184 184 317 317 317 F F F 19 380-399 184 184 184 317 317 317 F F F 20 400-419 183 183 183 314 314 314 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 185 185 185 318 318 318 F F F 23 460-479 186 186 186 319 319 319 F F F 24 480-499 186 186 186 319 319 319 T T T 25 500-519 186 186 186 319 319 319 F F F 26 520-539 186 186 186 319 319 319 T T T 27 540-559 184 184 184 316 316 316 F F F 28 560-579 183 183 183 314 314 314 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 186, Max flush: 321 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 187 187 187 322 322 322 F F F 2 040-059 181 181 181 311 311 311 F F F 3 060-079 187 187 187 322 322 322 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 186 186 186 319 319 319 F F F 6 120-139 185 185 185 318 318 318 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 186 186 186 319 319 319 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 185 185 185 318 318 318 F F F 12 240-259 184 184 184 317 317 317 F F F 13 260-279 185 185 185 318 318 318 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 186 186 186 319 319 319 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 184 184 184 317 317 317 F F F 19 380-399 184 184 184 317 317 317 F F F 20 400-419 183 183 183 314 314 314 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 185 185 185 318 318 318 F F F 23 460-479 186 186 186 319 319 319 F F F 24 480-499 187 187 187 322 322 322 F F F 25 500-519 186 186 186 319 319 319 F F F 26 520-539 187 187 187 322 322 322 F F F 27 540-559 184 184 184 316 316 316 F F F 28 560-579 183 183 183 314 314 314 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 187, Max flush: 322 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.793361888Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:20.835873506Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.836232346Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.868035163Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.868632151Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.868667466Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.868682254Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.868696752Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933} {"msg":"Crucible 92209d45-be3a-4c19-84df-42c6bb9ba6e0 has session id: 29f08504-f385-484f-93fa-910f5bb6c51d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.869127232Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Upstairs opts: Upstairs UUID: 92209d45-be3a-4c19-84df-42c6bb9ba6e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.869173006Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Crucible stats registered with UUID: 92209d45-be3a-4c19-84df-42c6bb9ba6e0","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.869197003Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900046581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933} {"msg":"92209d45-be3a-4c19-84df-42c6bb9ba6e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900082946Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900112312Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900158716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900196871Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900231576Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900268121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900301906Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900337581Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"io task","client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900374656Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"io task","client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900407062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"io task","client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57795)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900440557Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"io task","client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50811)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.900519276Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"io task","client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43195)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.90056162Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"io task","client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.901258384Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.90128583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.901308767Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.901322865Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.901414942Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.901447108Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.901948669Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902044085Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902112716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902480375Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902504582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.90251867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902618476Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902637334Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902654881Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.90273291Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902752998Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[0]R flush_numbers[0..12]: [320, 322, 311, 322, 303, 319, 318, 317, 311, 319, 321, 318]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902772975Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[0]R generation[0..12]: [186, 187, 181, 187, 176, 186, 185, 184, 181, 186, 186, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902787273Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902801291Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[1]R flush_numbers[0..12]: [320, 322, 311, 322, 303, 319, 318, 317, 311, 319, 321, 318]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902815159Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[1]R generation[0..12]: [186, 187, 181, 187, 176, 186, 185, 184, 181, 186, 186, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902828757Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902842735Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[2]R flush_numbers[0..12]: [320, 322, 311, 322, 303, 319, 318, 317, 311, 319, 321, 318]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902856573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[2]R generation[0..12]: [186, 187, 181, 187, 176, 186, 185, 184, 181, 186, 186, 185]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902870121Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.90288402Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902897078Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902911496Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902927663Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902940952Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"":"downstairs","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.90295409Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.902986405Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"0","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.903001163Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"1","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.903014841Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"client":"2","":"downstairs","session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.90302818Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.903040548Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"92209d45-be3a-4c19-84df-42c6bb9ba6e0 is now active with session: 29f08504-f385-484f-93fa-910f5bb6c51d","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.903053356Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.903065424Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933,"session_id":"29f08504-f385-484f-93fa-910f5bb6c51d"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:20.903077753Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 573 len 24576 data: 15 9 7 7 10 10 02/30 Write block 83 len 36864 data: 12 13 15 14 15 17 16 13 13 03/30 Read block 571 len 32768 04/30 Read block 457 len 4096 05/30 Read block 5 len 20480 06/30 Read block 289 len 16384 07/30 Read block 297 len 32768 08/30 Flush 09/30 Write block 227 len 24576 data: 21 22 20 20 19 14 10/30 Write block 120 len 4096 data: 18 11/30 Read block 53 len 28672 12/30 Read block 79 len 8192 13/30 Read block 259 len 8192 14/30 Write block 493 len 32768 data: 9 9 9 12 13 12 12 12 15/30 Read block 394 len 20480 16/30 Flush 17/30 Write block 532 len 16384 data: 19 19 17 15 18/30 Write block 402 len 40960 data: 13 13 15 17 17 15 16 18 16 15 19/30 Write block 240 len 40960 data: 13 9 10 13 15 16 21 25 23 22 20/30 Write block 67 len 36864 data: 26 21 23 23 23 21 21 16 14 21/30 Write block 455 len 28672 data: 15 17 19 21 21 20 19 22/30 Write block 415 len 40960 data: 12 13 14 17 14 11 11 10 7 10 23/30 Read block 13 len 24576 24/30 Write block 33 len 32768 data: 13 11 8 9 10 13 14 12 25/30 Write block 379 len 24576 data: 24 21 19 19 19 17 26/30 Write block 118 len 4096 data: 16 27/30 Write block 95 len 40960 data: 12 12 11 13 15 16 13 11 8 9 28/30 Write block 389 len 24576 data: 15 18 13 11 8 9 29/30 Read block 497 len 8192 30/30 Read block 347 len 24576 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Write 10 Done Done Done false 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 6 Done Sent Done false 29 Acked 1028 Read 2 Done Sent Done false 30 Acked 1029 Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 3 0 3 Done 14 11 14 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:20.925331649Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1933} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 187 187 187 322 322 322 T T T 2 040-059 181 181 181 311 311 311 T T T 3 060-079 187 187 187 322 322 322 T T T 4 080-099 188 188 188 323 323 323 T T T 5 100-119 186 186 186 319 319 319 T T T 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 186 186 186 319 319 319 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 188 188 188 324 324 324 F F F 12 240-259 184 184 184 317 317 317 T T T 13 260-279 185 185 185 318 318 318 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 186 186 186 319 319 319 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 184 184 184 317 317 317 T T T 19 380-399 184 184 184 317 317 317 T T T 20 400-419 183 183 183 314 314 314 T T T 21 420-439 184 184 184 315 315 315 T T T 22 440-459 185 185 185 318 318 318 T T T 23 460-479 186 186 186 319 319 319 T T T 24 480-499 188 188 188 324 324 324 F F F 25 500-519 188 188 188 324 324 324 F F F 26 520-539 187 187 187 322 322 322 T T T 27 540-559 184 184 184 316 316 316 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 188, Max flush: 324 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 189 189 189 325 325 325 F F F 3 060-079 189 189 189 325 325 325 F F F 4 080-099 189 189 189 325 325 325 F F F 5 100-119 189 189 189 325 325 325 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 181 181 181 311 311 311 F F F 9 180-199 186 186 186 319 319 319 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 188 188 188 324 324 324 F F F 12 240-259 189 189 189 325 325 325 F F F 13 260-279 185 185 185 318 318 318 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 186 186 186 319 319 319 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 189 189 189 325 325 325 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 189 189 189 325 325 325 F F F 22 440-459 189 189 189 325 325 325 F F F 23 460-479 189 189 189 325 325 325 F F F 24 480-499 188 188 188 324 324 324 F F F 25 500-519 188 188 188 324 324 324 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 184 184 184 316 316 316 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 189, Max flush: 325 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.373585304Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.418850531Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.419681646Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.41972873Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.419746377Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.419764575Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941} {"msg":"Crucible 0dd3ade0-8882-4cc0-9fde-035da7adba86 has session id: 0db669bd-1def-45d4-ab6b-2da1138eb676","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.420190586Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Upstairs opts: Upstairs UUID: 0dd3ade0-8882-4cc0-9fde-035da7adba86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.42023407Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Crucible stats registered with UUID: 0dd3ade0-8882-4cc0-9fde-035da7adba86","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.420264336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452263516Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941} {"msg":"0dd3ade0-8882-4cc0-9fde-035da7adba86 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45230867Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452324417Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452350034Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452365582Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45237975Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452393308Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452406956Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452420204Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"io task","client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452444371Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"io task","client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452470597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"io task","client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_connection connected from Ok(127.0.0.1:43971)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452681838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"io task","client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_connection connected from Ok(127.0.0.1:53434)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452722912Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"io task","client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_connection connected from Ok(127.0.0.1:62281)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.452777745Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"io task","client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.453494536Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.453523442Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45353857Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.453552578Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45367922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.453732623Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.454306263Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.454344758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.454374274Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.454891593Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.454929977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.454958944Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45498352Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455006977Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455028954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45505444Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455079027Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[0]R flush_numbers[0..12]: [320, 325, 325, 325, 325, 325, 324, 317, 311, 319, 321, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455103324Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[0]R generation[0..12]: [186, 189, 189, 189, 189, 189, 188, 184, 181, 186, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45512686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455149917Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[1]R flush_numbers[0..12]: [320, 325, 325, 325, 325, 325, 324, 317, 311, 319, 321, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455173924Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[1]R generation[0..12]: [186, 189, 189, 189, 189, 189, 188, 184, 181, 186, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45519936Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455226747Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[2]R flush_numbers[0..12]: [320, 325, 325, 325, 325, 325, 324, 317, 311, 319, 321, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455250243Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[2]R generation[0..12]: [186, 189, 189, 189, 189, 189, 188, 184, 181, 186, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45527519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455304416Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455327102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45534178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:57:25.455354709Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455375526Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"":"downstairs","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455389274Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455401432Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"0","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455414741Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"1","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455427719Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"client":"2","":"downstairs","session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455440807Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455452925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"0dd3ade0-8882-4cc0-9fde-035da7adba86 is now active with session: 0db669bd-1def-45d4-ab6b-2da1138eb676","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455465723Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.455477772Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941,"session_id":"0db669bd-1def-45d4-ab6b-2da1138eb676"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:25.45548993Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 5 len 24576 data: 8 10 13 16 15 17 02/30 Flush 03/30 Write block 502 len 28672 data: 14 14 11 8 10 10 9 04/30 Read block 291 len 4096 05/30 Read block 565 len 40960 06/30 Read block 62 len 32768 07/30 Flush 08/30 Write block 467 len 20480 data: 14 14 15 16 17 09/30 Write block 164 len 12288 data: 10 12 10 10/30 Write block 423 len 20480 data: 8 11 11 13 12 11/30 Read block 236 len 24576 12/30 Read block 480 len 24576 13/30 Read block 101 len 32768 14/30 Write block 284 len 32768 data: 13 13 14 12 10 10 10 10 15/30 Read block 568 len 36864 16/30 Write block 177 len 16384 data: 13 14 14 13 17/30 Read block 171 len 24576 18/30 Read block 557 len 40960 19/30 Read block 207 len 8192 20/30 Read block 38 len 40960 21/30 Read block 296 len 40960 22/30 Write block 208 len 4096 data: 14 23/30 Read block 140 len 40960 24/30 Read block 212 len 40960 25/30 Read block 345 len 16384 26/30 Write block 249 len 20480 data: 23 18 17 16 13 27/30 Read block 7 len 28672 28/30 Read block 472 len 40960 29/30 Read block 275 len 16384 30/30 Write block 389 len 16384 data: 16 19 14 12 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 8 Acked 1007 Write 5 Done Done Done false 9 Acked 1008 Write 3 Done Done Done false 10 Acked 1009 Write 5 Done Done Done false 11 Acked 1010 Read 6 Done Done Done false 12 Acked 1011 Read 6 Done Done Done false 13 Acked 1012 Read 8 Done Done Done false 14 Acked 1013 Write 8 Done Done Done false 15 Acked 1014 Read 9 Done Done Done false 16 Acked 1015 Write 4 Done Done Done false 17 Acked 1016 Read 6 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:25.480337338Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1941} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 326 326 326 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 189 189 189 325 325 325 F F F 3 060-079 189 189 189 325 325 325 F F F 4 080-099 189 189 189 325 325 325 F F F 5 100-119 189 189 189 325 325 325 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 181 181 181 311 311 311 T T T 9 180-199 186 186 186 319 319 319 T T T 10 200-219 186 186 186 321 321 321 T T T 11 220-239 188 188 188 324 324 324 F F F 12 240-259 189 189 189 325 325 325 T T T 13 260-279 185 185 185 318 318 318 F F F 14 280-299 178 178 178 307 307 307 T T T 15 300-319 186 186 186 319 319 319 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 189 189 189 325 325 325 T T T 20 400-419 189 189 189 325 325 325 F F F 21 420-439 189 189 189 325 325 325 T T T 22 440-459 189 189 189 325 325 325 F F F 23 460-479 189 189 189 325 325 325 T T T 24 480-499 188 188 188 324 324 324 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 184 184 184 316 316 316 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 190, Max flush: 327 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 326 326 326 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 189 189 189 325 325 325 F F F 3 060-079 189 189 189 325 325 325 F F F 4 080-099 189 189 189 325 325 325 F F F 5 100-119 189 189 189 325 325 325 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 191 191 191 328 328 328 F F F 9 180-199 191 191 191 328 328 328 F F F 10 200-219 191 191 191 328 328 328 F F F 11 220-239 188 188 188 324 324 324 F F F 12 240-259 191 191 191 328 328 328 F F F 13 260-279 185 185 185 318 318 318 F F F 14 280-299 191 191 191 328 328 328 F F F 15 300-319 186 186 186 319 319 319 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 191 191 191 328 328 328 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 191 191 191 328 328 328 F F F 22 440-459 189 189 189 325 325 325 F F F 23 460-479 191 191 191 328 328 328 F F F 24 480-499 188 188 188 324 324 324 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 184 184 184 316 316 316 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 191, Max flush: 328 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.844510357Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:29.887906072Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.888263203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.920173105Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.920881037Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.920916092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.92093425Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.920953317Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949} {"msg":"Crucible ccf574b2-f86f-43e8-a4c8-b8b0c1b76e9b has session id: 40284314-76da-4560-b381-209eecfabdb5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.923569896Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Upstairs opts: Upstairs UUID: ccf574b2-f86f-43e8-a4c8-b8b0c1b76e9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.923600102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Crucible stats registered with UUID: ccf574b2-f86f-43e8-a4c8-b8b0c1b76e9b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.923616699Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.95437897Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949} {"msg":"ccf574b2-f86f-43e8-a4c8-b8b0c1b76e9b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954408806Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954425763Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.95445428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954473567Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954490794Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954507362Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.95452455Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954541387Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"io task","client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954558645Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"io task","client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954604519Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"io task","client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37087)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954692437Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"io task","client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44983)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954909897Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"io task","client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47429)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.954950691Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"io task","client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.955592102Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.955628197Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.955661143Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.955675421Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.955688949Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.955702397Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.956290996Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.956401061Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.956421248Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.956900692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.956941046Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.956970192Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957041922Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957111372Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.95712942Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957142598Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957156686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[0]R flush_numbers[0..12]: [326, 325, 325, 325, 325, 325, 324, 317, 328, 328, 328, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957170934Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[0]R generation[0..12]: [190, 189, 189, 189, 189, 189, 188, 184, 191, 191, 191, 188]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957184702Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.95719866Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[1]R flush_numbers[0..12]: [326, 325, 325, 325, 325, 325, 324, 317, 328, 328, 328, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957212569Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[1]R generation[0..12]: [190, 189, 189, 189, 189, 189, 188, 184, 191, 191, 191, 188]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957238635Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957253203Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[2]R flush_numbers[0..12]: [326, 325, 325, 325, 325, 325, 324, 317, 328, 328, 328, 324]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957266991Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[2]R generation[0..12]: [190, 189, 189, 189, 189, 189, 188, 184, 191, 191, 191, 188]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957280589Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957294457Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957307345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957322933Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957336352Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957359788Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"":"downstairs","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957373266Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957385345Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"0","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957398423Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"1","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957411361Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"client":"2","":"downstairs","session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957424499Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957436718Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"ccf574b2-f86f-43e8-a4c8-b8b0c1b76e9b is now active with session: 40284314-76da-4560-b381-209eecfabdb5","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957449636Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957461464Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949,"session_id":"40284314-76da-4560-b381-209eecfabdb5"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:29.957473682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 310 len 20480 data: 13 13 14 13 14 02/30 Read block 532 len 8192 03/30 Read block 381 len 8192 04/30 Read block 544 len 8192 05/30 Read block 65 len 32768 06/30 Write block 238 len 16384 data: 12 14 14 10 07/30 Read block 449 len 40960 08/30 Read block 563 len 28672 09/30 Write block 471 len 36864 data: 18 16 12 14 12 13 12 13 18 10/30 Write block 163 len 12288 data: 10 11 13 11/30 Write block 468 len 40960 data: 15 16 17 19 17 13 15 13 14 13 12/30 Read block 575 len 32768 13/30 Read block 175 len 28672 14/30 Write block 288 len 12288 data: 11 11 11 15/30 Flush 16/30 Write block 302 len 40960 data: 15 13 17 19 18 16 14 11 14 14 17/30 Read block 166 len 4096 18/30 Write block 402 len 40960 data: 14 14 16 18 18 16 17 19 17 16 19/30 Read block 317 len 12288 20/30 Read block 292 len 20480 21/30 Flush 22/30 Write block 450 len 24576 data: 12 12 13 17 15 16 23/30 Read block 470 len 40960 24/30 Write block 215 len 12288 data: 10 10 11 25/30 Write block 87 len 24576 data: 16 18 17 14 14 13 26/30 Read block 237 len 12288 27/30 Read block 477 len 40960 28/30 Read block 355 len 20480 29/30 Read block 12 len 4096 30/30 Read block 256 len 40960 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:29.98319344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1949} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 326 326 326 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 189 189 189 325 325 325 F F F 3 060-079 189 189 189 325 325 325 F F F 4 080-099 189 189 189 325 325 325 T T T 5 100-119 189 189 189 325 325 325 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 192 192 192 329 329 329 F F F 9 180-199 191 191 191 328 328 328 F F F 10 200-219 191 191 191 328 328 328 T T T 11 220-239 192 192 192 329 329 329 F F F 12 240-259 192 192 192 329 329 329 F F F 13 260-279 185 185 185 318 318 318 F F F 14 280-299 192 192 192 329 329 329 F F F 15 300-319 192 192 192 330 330 330 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 191 191 191 328 328 328 F F F 20 400-419 192 192 192 330 330 330 F F F 21 420-439 191 191 191 328 328 328 F F F 22 440-459 189 189 189 325 325 325 T T T 23 460-479 192 192 192 329 329 329 F F F 24 480-499 188 188 188 324 324 324 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 184 184 184 316 316 316 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 192, Max flush: 330 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 326 326 326 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 189 189 189 325 325 325 F F F 3 060-079 189 189 189 325 325 325 F F F 4 080-099 193 193 193 331 331 331 F F F 5 100-119 189 189 189 325 325 325 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 192 192 192 329 329 329 F F F 9 180-199 191 191 191 328 328 328 F F F 10 200-219 193 193 193 331 331 331 F F F 11 220-239 192 192 192 329 329 329 F F F 12 240-259 192 192 192 329 329 329 F F F 13 260-279 185 185 185 318 318 318 F F F 14 280-299 192 192 192 329 329 329 F F F 15 300-319 192 192 192 330 330 330 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 191 191 191 328 328 328 F F F 20 400-419 192 192 192 330 330 330 F F F 21 420-439 191 191 191 328 328 328 F F F 22 440-459 193 193 193 331 331 331 F F F 23 460-479 192 192 192 329 329 329 F F F 24 480-499 188 188 188 324 324 324 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 184 184 184 316 316 316 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 193, Max flush: 331 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.310641963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.355807974Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.356663676Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.356700391Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.356715058Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.356729367Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957} {"msg":"Crucible 56f4a1c8-595a-4ddd-af00-5c817672138e has session id: f66089f0-14ac-4044-bee1-eb824298c676","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.357229727Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Upstairs opts: Upstairs UUID: 56f4a1c8-595a-4ddd-af00-5c817672138e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.357265073Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Crucible stats registered with UUID: 56f4a1c8-595a-4ddd-af00-5c817672138e","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.35728597Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.388947436Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957} {"msg":"56f4a1c8-595a-4ddd-af00-5c817672138e active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389027405Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389047062Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389086177Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389121692Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389149318Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389171855Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389203431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389235196Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"io task","client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389267442Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"io task","client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389286319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"io task","client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_connection connected from Ok(127.0.0.1:52207)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389304817Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"io task","client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_connection connected from Ok(127.0.0.1:32936)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389322244Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"io task","client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_connection connected from Ok(127.0.0.1:44192)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.389628122Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"io task","client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390125183Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390156009Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390172297Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390197703Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390457628Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390484404Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390952509Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.390988594Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.39101852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391543358Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391582142Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391610058Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391704395Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391732431Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391747929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391766287Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391780625Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[0]R flush_numbers[0..12]: [326, 325, 325, 325, 331, 325, 324, 317, 329, 328, 331, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391794983Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[0]R generation[0..12]: [190, 189, 189, 189, 193, 189, 188, 184, 192, 191, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391808791Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391822929Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[1]R flush_numbers[0..12]: [326, 325, 325, 325, 331, 325, 324, 317, 329, 328, 331, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391836927Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[1]R generation[0..12]: [190, 189, 189, 189, 193, 189, 188, 184, 192, 191, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391850705Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391864583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[2]R flush_numbers[0..12]: [326, 325, 325, 325, 331, 325, 324, 317, 329, 328, 331, 329]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391878221Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[2]R generation[0..12]: [190, 189, 189, 189, 193, 189, 188, 184, 192, 191, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391891759Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391905757Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391918935Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391932264Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391945302Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391967769Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"":"downstairs","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391981307Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.391993435Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"0","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.392006854Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"1","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.392020162Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"client":"2","":"downstairs","session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.39203358Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.392046058Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"56f4a1c8-595a-4ddd-af00-5c817672138e is now active with session: f66089f0-14ac-4044-bee1-eb824298c676","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.392058806Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.392070695Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957,"session_id":"f66089f0-14ac-4044-bee1-eb824298c676"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:34.392082853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 94 len 32768 data: 11 13 13 12 14 16 17 14 02/30 Write block 107 len 40960 data: 11 13 11 12 9 9 8 7 13 14 03/30 Write block 545 len 8192 data: 12 12 04/30 Read block 449 len 12288 05/30 Read block 185 len 12288 06/30 Read block 128 len 4096 07/30 Write block 275 len 24576 data: 17 17 18 19 19 17 08/30 Read block 49 len 28672 09/30 Write block 82 len 40960 data: 10 13 14 16 15 17 19 18 15 15 10/30 Write block 220 len 32768 data: 13 16 16 13 16 17 19 22 11/30 Read block 456 len 28672 12/30 Flush 13/30 Write block 295 len 24576 data: 11 15 17 18 17 16 14/30 Write block 306 len 4096 data: 19 15/30 Read block 68 len 20480 16/30 Read block 195 len 12288 17/30 Read block 499 len 20480 18/30 Read block 346 len 36864 19/30 Write block 100 len 40960 data: 18 15 12 9 10 10 11 12 14 12 20/30 Read block 243 len 32768 21/30 Read block 484 len 36864 22/30 Write block 459 len 24576 data: 22 21 20 16 18 18 23/30 Write block 489 len 28672 data: 11 12 10 7 10 10 10 24/30 Read block 430 len 20480 25/30 Read block 489 len 4096 26/30 Read block 498 len 40960 27/30 Write block 119 len 4096 data: 18 28/30 Read block 414 len 4096 29/30 Read block 554 len 28672 30/30 Write block 42 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 6 Done Done Done false 14 Acked 1013 Write 1 Done Done Done false 15 Acked 1014 Read 5 Done Done Done false 16 Acked 1015 Read 3 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Read 9 Done Done Done false 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Read 8 Done Done Done false 21 Acked 1020 Read 9 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:34.415091925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1957} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 326 326 326 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 189 189 189 325 325 325 T T T 3 060-079 189 189 189 325 325 325 F F F 4 080-099 194 194 194 332 332 332 F F F 5 100-119 194 194 194 332 332 332 T T T 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 192 192 192 329 329 329 F F F 9 180-199 191 191 191 328 328 328 F F F 10 200-219 193 193 193 331 331 331 F F F 11 220-239 194 194 194 332 332 332 F F F 12 240-259 192 192 192 329 329 329 F F F 13 260-279 194 194 194 332 332 332 F F F 14 280-299 194 194 194 332 332 332 T T T 15 300-319 192 192 192 330 330 330 T T T 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 191 191 191 328 328 328 F F F 20 400-419 192 192 192 330 330 330 F F F 21 420-439 191 191 191 328 328 328 F F F 22 440-459 193 193 193 331 331 331 T T T 23 460-479 192 192 192 329 329 329 T T T 24 480-499 188 188 188 324 324 324 T T T 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 194 194 194 332 332 332 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 194, Max flush: 332 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 326 326 326 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 195 195 195 333 333 333 F F F 3 060-079 189 189 189 325 325 325 F F F 4 080-099 194 194 194 332 332 332 F F F 5 100-119 195 195 195 333 333 333 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 192 192 192 329 329 329 F F F 9 180-199 191 191 191 328 328 328 F F F 10 200-219 193 193 193 331 331 331 F F F 11 220-239 194 194 194 332 332 332 F F F 12 240-259 192 192 192 329 329 329 F F F 13 260-279 194 194 194 332 332 332 F F F 14 280-299 195 195 195 333 333 333 F F F 15 300-319 195 195 195 333 333 333 F F F 16 320-339 186 186 186 320 320 320 F F F 17 340-359 181 181 181 311 311 311 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 191 191 191 328 328 328 F F F 20 400-419 192 192 192 330 330 330 F F F 21 420-439 191 191 191 328 328 328 F F F 22 440-459 195 195 195 333 333 333 F F F 23 460-479 195 195 195 333 333 333 F F F 24 480-499 195 195 195 333 333 333 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 194 194 194 332 332 332 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 195, Max flush: 333 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.763269454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.809640208Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.810462274Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.810506178Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.810523546Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.810541583Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965} {"msg":"Crucible 6d099d4b-83bb-4fd7-b740-bf737bb89e2f has session id: df64db0d-c467-4249-9fc2-65647447582b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.810984022Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Upstairs opts: Upstairs UUID: 6d099d4b-83bb-4fd7-b740-bf737bb89e2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.811021787Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Crucible stats registered with UUID: 6d099d4b-83bb-4fd7-b740-bf737bb89e2f","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.811045694Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843176925Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965} {"msg":"6d099d4b-83bb-4fd7-b740-bf737bb89e2f active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843226428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843256544Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843307127Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843345262Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843418702Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843453737Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843485453Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843514289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"io task","client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843550373Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"io task","client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843585369Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"io task","client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60702)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843616784Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"io task","client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38246)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.84364945Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"io task","client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43735)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.843680746Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"io task","client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.844511461Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.844543106Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.844559704Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.844580771Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.844606518Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.844632454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845153712Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.84531215Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845350255Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845729492Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845755429Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845770417Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845868483Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.84588807Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845902109Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845959071Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.845983297Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[0]R flush_numbers[0..12]: [326, 325, 333, 325, 332, 333, 324, 317, 329, 328, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846002675Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[0]R generation[0..12]: [190, 189, 195, 189, 194, 195, 188, 184, 192, 191, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846016633Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846030761Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[1]R flush_numbers[0..12]: [326, 325, 333, 325, 332, 333, 324, 317, 329, 328, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846044689Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[1]R generation[0..12]: [190, 189, 195, 189, 194, 195, 188, 184, 192, 191, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846058447Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846072365Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[2]R flush_numbers[0..12]: [326, 325, 333, 325, 332, 333, 324, 317, 329, 328, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846085913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[2]R generation[0..12]: [190, 189, 195, 189, 194, 195, 188, 184, 192, 191, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846099501Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846113119Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846126108Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846139306Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846152134Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846164912Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"":"downstairs","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.84618005Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-09-27T03:57:38.846192319Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"0","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846217985Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"1","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846243632Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"client":"2","":"downstairs","session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846265948Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846287835Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"6d099d4b-83bb-4fd7-b740-bf737bb89e2f is now active with session: df64db0d-c467-4249-9fc2-65647447582b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846310612Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:38.846332599Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965,"session_id":"df64db0d-c467-4249-9fc2-65647447582b"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30region info: , block size: 4096 bytes "time":"2024-09-27T03:57:38.846354476Z" sub_volume 0 blocks / extent: 20 ,"hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965} sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 498 len 4096 data: 13 02/30 Read block 504 len 12288 03/30 Write block 329 len 32768 data: 15 15 13 14 16 16 15 14 04/30 Read block 408 len 40960 05/30 Read block 300 len 8192 06/30 Read block 340 len 4096 07/30 Read block 119 len 32768 08/30 Read block 396 len 16384 09/30 Read block 216 len 12288 10/30 Read block 32 len 16384 11/30 Read block 251 len 32768 12/30 Read block 29 len 8192 13/30 Read block 343 len 12288 14/30 Read block 124 len 40960 15/30 Write block 1 len 16384 data: 3 8 9 8 16/30 Read block 449 len 4096 17/30 Write block 195 len 8192 data: 18 18 18/30 Write block 242 len 24576 data: 11 14 16 17 22 26 19/30 Read block 544 len 16384 20/30 Read block 408 len 28672 21/30 Read block 334 len 28672 22/30 Read block 188 len 32768 23/30 Write block 93 len 32768 data: 12 12 14 14 13 15 17 19 24/30 Write block 52 len 4096 data: 14 25/30 Write block 184 len 32768 data: 14 12 13 11 12 13 17 15 26/30 Write block 248 len 32768 data: 24 24 19 18 17 14 13 14 27/30 Write block 40 len 28672 data: 13 15 17 17 16 18 19 28/30 Read block 465 len 12288 29/30 Write block 340 len 36864 data: 11 13 13 10 12 10 15 16 18 30/30 Write block 435 len 24576 data: 8 7 7 8 7 8 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 1 Done Done Done false 2 Acked 1001 Read 3 Done Done Done false 3 Acked 1002 Write 8 Done Done Done false 4 Acked 1003 Read 10 Done Done Done false 5 Acked 1004 Read 2 Done Done Done false 6 Acked 1005 Read 1 Done Done Done false 7 Acked 1006 Read 8 Done Done Done false 8 Acked 1007 Read 4 Done Done Done false 9 Acked 1008 Read 3 Done Done Done false 10 Acked 1009 Read 4 Done Done Done false 11 Acked 1010 Read 8 Done Done Done false 12 Acked 1011 Read 2 Done Done Done false 13 Acked 1012 Read 3 Done Done Done false 14 Acked 1013 Read 10 Done Done Done false 15 Acked 1014 Write 4 Done Done Done false 16 Acked 1015 Read 1 Done Done Done false 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Read 4 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 9 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:38.865896177Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1965} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 326 326 326 T T T 1 020-039 189 189 189 325 325 325 F F F 2 040-059 195 195 195 333 333 333 T T T 3 060-079 189 189 189 325 325 325 F F F 4 080-099 194 194 194 332 332 332 T T T 5 100-119 195 195 195 333 333 333 T T T 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 192 192 192 329 329 329 F F F 9 180-199 191 191 191 328 328 328 T T T 10 200-219 193 193 193 331 331 331 F F F 11 220-239 194 194 194 332 332 332 F F F 12 240-259 192 192 192 329 329 329 T T T 13 260-279 194 194 194 332 332 332 F F F 14 280-299 195 195 195 333 333 333 F F F 15 300-319 195 195 195 333 333 333 F F F 16 320-339 186 186 186 320 320 320 T T T 17 340-359 181 181 181 311 311 311 T T T 18 360-379 189 189 189 325 325 325 F F F 19 380-399 191 191 191 328 328 328 F F F 20 400-419 192 192 192 330 330 330 F F F 21 420-439 191 191 191 328 328 328 T T T 22 440-459 195 195 195 333 333 333 T T T 23 460-479 195 195 195 333 333 333 F F F 24 480-499 195 195 195 333 333 333 T T T 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 194 194 194 332 332 332 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 195, Max flush: 333 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 334 334 334 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 196 196 196 334 334 334 F F F 3 060-079 189 189 189 325 325 325 F F F 4 080-099 196 196 196 334 334 334 F F F 5 100-119 196 196 196 334 334 334 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 184 184 184 317 317 317 F F F 8 160-179 192 192 192 329 329 329 F F F 9 180-199 196 196 196 334 334 334 F F F 10 200-219 193 193 193 331 331 331 F F F 11 220-239 194 194 194 332 332 332 F F F 12 240-259 196 196 196 334 334 334 F F F 13 260-279 194 194 194 332 332 332 F F F 14 280-299 195 195 195 333 333 333 F F F 15 300-319 195 195 195 333 333 333 F F F 16 320-339 196 196 196 334 334 334 F F F 17 340-359 196 196 196 334 334 334 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 191 191 191 328 328 328 F F F 20 400-419 192 192 192 330 330 330 F F F 21 420-439 196 196 196 334 334 334 F F F 22 440-459 196 196 196 334 334 334 F F F 23 460-479 195 195 195 333 333 333 F F F 24 480-499 196 196 196 334 334 334 F F F 25 500-519 190 190 190 327 327 327 F F F 26 520-539 189 189 189 325 325 325 F F F 27 540-559 194 194 194 332 332 332 F F F 28 560-579 188 188 188 323 323 323 F F F 29 580-599 184 184 184 316 316 316 F F F Max gen: 196, Max flush: 334 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.262697289Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.309116637Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.310095332Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6901b9ee7172d672adc2780a2da758752eb0a2a4\",\n git_commit_timestamp: \"2024-09-27T03:30:33.000000000Z\",\n git_branch: \"alan/more-dsc-for-tests\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.310141156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.310158853Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.31017716Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973} {"msg":"Crucible abef47b4-18a5-4561-8fbc-73e491590783 has session id: 8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.31061783Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Upstairs opts: Upstairs UUID: abef47b4-18a5-4561-8fbc-73e491590783, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.310665573Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Crucible stats registered with UUID: abef47b4-18a5-4561-8fbc-73e491590783","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.310694099Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343136418Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973} {"msg":"abef47b4-18a5-4561-8fbc-73e491590783 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343182921Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343198969Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343220366Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343234344Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343247682Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.34326065Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343274129Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343287177Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343310164Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.34333864Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58034)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.3434831Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43488)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.343528613Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63482)","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.34355774Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.34435291Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.344381156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.344408852Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.34442358Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.344437348Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.344450686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345045454Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345224449Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345259485Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345597758Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345635913Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345661689Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345737838Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345761615Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345775613Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.3458004Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345814268Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[0]R flush_numbers[0..12]: [334, 325, 334, 325, 334, 334, 324, 317, 329, 334, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345879579Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[0]R generation[0..12]: [196, 189, 196, 189, 196, 196, 188, 184, 192, 196, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345897686Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345911954Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[1]R flush_numbers[0..12]: [334, 325, 334, 325, 334, 334, 324, 317, 329, 334, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345926092Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[1]R generation[0..12]: [196, 189, 196, 189, 196, 196, 188, 184, 192, 196, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.34594007Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345954298Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[2]R flush_numbers[0..12]: [334, 325, 334, 325, 334, 334, 324, 317, 329, 334, 331, 332]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345967927Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[2]R generation[0..12]: [196, 189, 196, 189, 196, 196, 188, 184, 192, 196, 193, 194]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.345996843Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.34601155Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Max found gen is 197","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346024779Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Generation requested: 198 >= found:197","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346040897Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346054185Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346067213Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"":"downstairs","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346080191Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346092089Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"0","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2024-09-27T03:57:43.346113156Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"1","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346126625Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"client":"2","":"downstairs","session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346139963Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346152101Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"abef47b4-18a5-4561-8fbc-73e491590783 is now active with session: 8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346164919Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346176908Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973,"session_id":"8dcf30a0-aa5c-4c00-8d36-aaeabe3acc9b"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-09-27T03:57:43.346189336Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973} region info: block size: 4096 bytes sub_volume 0 blocks / extent: 20 sub_volume 0 extent size: 80 KiB sub_volume 0 extent count: 30 total blocks: 600 total size: 2.3 MiB encryption: no Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 40960 data: 12 12 12 14 15 15 17 19 19 17 02/30 Write block 431 len 36864 data: 12 16 12 11 9 8 8 9 8 03/30 Write block 84 len 24576 data: 15 17 16 18 20 19 04/30 Read block 5 len 12288 05/30 Write block 570 len 36864 data: 16 15 14 16 10 8 8 11 11 06/30 Write block 401 len 36864 data: 15 16 16 18 20 20 18 18 20 07/30 Read block 10 len 36864 08/30 Write block 417 len 28672 data: 15 18 15 12 12 11 9 09/30 Read block 536 len 20480 10/30 Read block 453 len 16384 11/30 Read block 240 len 8192 12/30 Read block 460 len 36864 13/30 Read block 214 len 24576 14/30 Write block 504 len 28672 data: 12 9 11 11 10 11 10 15/30 Write block 259 len 8192 data: 14 16 16/30 Read block 175 len 28672 17/30 Write block 534 len 28672 data: 18 16 18 14 13 9 13 18/30 Write block 251 len 12288 data: 19 18 15 19/30 Write block 154 len 32768 data: 14 12 13 14 17 16 14 14 20/30 Write block 594 len 4096 data: 11 21/30 Flush 22/30 Write block 57 len 28672 data: 14 12 10 12 13 13 16 23/30 Read block 189 len 16384 24/30 Write block 325 len 20480 data: 17 16 15 15 16 25/30 Read block 451 len 24576 26/30 Read block 588 len 12288 27/30 Write block 269 len 12288 data: 10 11 12 28/30 Write block 139 len 4096 data: 16 29/30 Read block 505 len 28672 30/30 Read block 488 len 4096 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T03:57:43.369775428Z","hostname":"w-01J8RSHFZRJY0A0NRJXKBWC2HA","pid":1973} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 334 334 334 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 196 196 196 334 334 334 T T T 3 060-079 189 189 189 325 325 325 T T T 4 080-099 198 198 198 335 335 335 F F F 5 100-119 196 196 196 334 334 334 F F F 6 120-139 188 188 188 324 324 324 T T T 7 140-159 198 198 198 335 335 335 F F F 8 160-179 198 198 198 335 335 335 F F F 9 180-199 196 196 196 334 334 334 F F F 10 200-219 193 193 193 331 331 331 F F F 11 220-239 194 194 194 332 332 332 F F F 12 240-259 198 198 198 335 335 335 F F F 13 260-279 198 198 198 335 335 335 T T T 14 280-299 195 195 195 333 333 333 F F F 15 300-319 195 195 195 333 333 333 F F F 16 320-339 196 196 196 334 334 334 T T T 17 340-359 196 196 196 334 334 334 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 198 198 198 335 335 335 F F F 20 400-419 198 198 198 335 335 335 F F F 21 420-439 198 198 198 335 335 335 F F F 22 440-459 196 196 196 334 334 334 F F F 23 460-479 195 195 195 333 333 333 F F F 24 480-499 196 196 196 334 334 334 F F F 25 500-519 198 198 198 335 335 335 F F F 26 520-539 198 198 198 335 335 335 F F F 27 540-559 198 198 198 335 335 335 F F F 28 560-579 198 198 198 335 335 335 F F F 29 580-599 198 198 198 335 335 335 F F F Max gen: 198, Max flush: 335 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 334 334 334 F F F 1 020-039 189 189 189 325 325 325 F F F 2 040-059 199 199 199 336 336 336 F F F 3 060-079 199 199 199 336 336 336 F F F 4 080-099 198 198 198 335 335 335 F F F 5 100-119 196 196 196 334 334 334 F F F 6 120-139 199 199 199 336 336 336 F F F 7 140-159 198 198 198 335 335 335 F F F 8 160-179 198 198 198 335 335 335 F F F 9 180-199 196 196 196 334 334 334 F F F 10 200-219 193 193 193 331 331 331 F F F 11 220-239 194 194 194 332 332 332 F F F 12 240-259 198 198 198 335 335 335 F F F 13 260-279 199 199 199 336 336 336 F F F 14 280-299 195 195 195 333 333 333 F F F 15 300-319 195 195 195 333 333 333 F F F 16 320-339 199 199 199 336 336 336 F F F 17 340-359 196 196 196 334 334 334 F F F 18 360-379 189 189 189 325 325 325 F F F 19 380-399 198 198 198 335 335 335 F F F 20 400-419 198 198 198 335 335 335 F F F 21 420-439 198 198 198 335 335 335 F F F 22 440-459 196 196 196 334 334 334 F F F 23 460-479 195 195 195 333 333 333 F F F 24 480-499 196 196 196 334 334 334 F F F 25 500-519 198 198 198 335 335 335 F F F 26 520-539 198 198 198 335 335 335 F F F 27 540-559 198 198 198 335 335 335 F F F 28 560-579 198 198 198 335 335 335 F F F 29 580-599 198 198 198 335 335 335 F F F Max gen: 199, Max flush: 336 7:30 Test duration Test completed real 7:30.416936861 user 57.243722456 sys 19.859096698 trap 0.074932206 tflt 0.038130655 dflt 0.111857602 kflt 0.006521986 lock 11:52:17.474454243 slp 36:25.695296563 lat 5.731630425 stop 0.061077648