Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.292015 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.022125935 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.023128562 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1192 1193 1194 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.541374136Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.620912543Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.622304176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.622374323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.62239067Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.622405287Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196} {"msg":"Crucible 0c40840b-9f85-4d42-8012-13f6bc90ec85 has session id: 89f710dc-ee43-4a91-8adc-4e68b54f66ac","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.623173045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Upstairs opts: Upstairs UUID: 0c40840b-9f85-4d42-8012-13f6bc90ec85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.6232017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Crucible stats registered with UUID: 0c40840b-9f85-4d42-8012-13f6bc90ec85","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.623216917Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:22.623276856Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624158643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624232899Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196} {"msg":"0c40840b-9f85-4d42-8012-13f6bc90ec85 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624308035Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.62433457Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"0","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624408307Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"0","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624424204Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"1","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624438911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"1","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624452619Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"2","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624466896Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"2","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624479714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624493141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624506269Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:57527)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624643373Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:50439)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624678847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:47502)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.624869422Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.62660752Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"2","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.626644123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"0","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.626677907Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"1","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.627828364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"1","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.627867437Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"2","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.627895462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"0","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.62892934Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629056987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629210119Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629231595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629246142Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629259579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629272687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629286025Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629299132Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629312Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629331106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629346933Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629369119Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629390725Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629413451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"":"downstairs","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629428538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:12:22.629449124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"0","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.6294743Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"1","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629491936Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"client":"2","":"downstairs","session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629505204Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.62952568Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"0c40840b-9f85-4d42-8012-13f6bc90ec85 is now active with session: 89f710dc-ee43-4a91-8adc-4e68b54f66ac","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629549496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629572791Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196,"session_id":"89f710dc-ee43-4a91-8adc-4e68b54f66ac"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:22.629594457Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1196} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.759085186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.809885228Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.810946242Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.810992783Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.811008061Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.811023668Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198} {"msg":"Crucible 04b6b570-5f1d-45bf-9173-f706a329f890 has session id: 84d83c25-48d0-4d93-842c-db5378384e4e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.811579115Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Upstairs opts: Upstairs UUID: 04b6b570-5f1d-45bf-9173-f706a329f890, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.811615048Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Crucible stats registered with UUID: 04b6b570-5f1d-45bf-9173-f706a329f890","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.811630555Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:26.811642783Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812059836Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812100288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198} {"msg":"04b6b570-5f1d-45bf-9173-f706a329f890 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812121904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.81214534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"0","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812192881Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"0","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812223216Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"1","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812249491Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"1","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812335615Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"2","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.8124694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"2","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812506963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812536398Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812562803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59315)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812590868Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60777)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812703267Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33787)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.812731942Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.813200155Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"2","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.813357406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"0","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.813841257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"2","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.813975312Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"0","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.814762766Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.814803108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.815186857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"1","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.815740295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"1","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816278096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816303281Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816318618Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816332496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816345743Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816359291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816372708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816391395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816405862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.81641869Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816431817Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816444515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816464751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"":"downstairs","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816477949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816489647Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"0","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816503064Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"1","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816516312Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"client":"2","":"downstairs","session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816529259Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816541147Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"04b6b570-5f1d-45bf-9173-f706a329f890 is now active with session: 84d83c25-48d0-4d93-842c-db5378384e4e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816553995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.816565722Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198,"session_id":"84d83c25-48d0-4d93-842c-db5378384e4e"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:26.81657777Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 104 len 8192 data: 2 2 02/30 Write block 497 len 16384 data: 2 2 2 2 03/30 Write block 453 len 24576 data: 2 2 2 2 2 2 04/30 Write block 320 len 20480 data: 2 2 2 2 2 05/30 Read block 27 len 16384 06/30 Read block 347 len 12288 07/30 Read block 39 len 24576 08/30 Read block 77 len 28672 09/30 Write block 10 len 28672 data: 2 2 2 2 2 2 2 10/30 Write block 138 len 8192 data: 2 2 11/30 Read block 92 len 16384 12/30 Read block 578 len 36864 13/30 Write block 108 len 32768 data: 2 2 2 2 2 2 2 2 14/30 Read block 180 len 16384 15/30 Read block 165 len 32768 16/30 Read block 101 len 12288 17/30 Flush 18/30 Write block 320 len 36864 data: 3 3 3 3 3 2 2 2 2 19/30 Read block 515 len 40960 20/30 Read block 266 len 20480 21/30 Read block 506 len 32768 22/30 Write block 145 len 16384 data: 2 2 2 2 23/30 Read block 593 len 16384 24/30 Write block 202 len 8192 data: 2 2 25/30 Read block 273 len 28672 26/30 Read block 147 len 12288 27/30 Write block 418 len 40960 data: 2 2 2 2 2 2 2 2 2 2 28/30 Write block 40 len 4096 data: 2 29/30 Read block 195 len 36864 30/30 Read block 176 len 20480 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:26.838399914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 2 2 2 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 T T T 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 2 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 3 3 3 3 3 3 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 3 3 3 3 3 3 F F F 21 420-439 3 3 3 3 3 3 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 3 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.102111824Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.150598705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.151608118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.15165036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.151664798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.151679095Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206} {"msg":"Crucible 91bfb1bd-24c0-4754-b9e5-8a64d2eb1f03 has session id: 1158eeaa-fc3e-4562-bec8-2a3b7901f612","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.152413809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Upstairs opts: Upstairs UUID: 91bfb1bd-24c0-4754-b9e5-8a64d2eb1f03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.152455941Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Crucible stats registered with UUID: 91bfb1bd-24c0-4754-b9e5-8a64d2eb1f03","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.152479017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:41.152500303Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.152913327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.152938652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206} {"msg":"91bfb1bd-24c0-4754-b9e5-8a64d2eb1f03 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153014758Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153034414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"0","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153061589Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"0","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153075187Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"1","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153098242Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"1","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153123418Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"2","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153148393Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"2","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153174008Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153200164Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153228388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"ds_connection connected from Ok(127.0.0.1:59194)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153254844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"ds_connection connected from Ok(127.0.0.1:47081)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153279169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"ds_connection connected from Ok(127.0.0.1:38801)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.153303364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.154010844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"0","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.154051766Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"2","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.154152727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"1","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.154647576Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"0","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.154699176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"2","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.154798328Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"1","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155177578Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155261862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[0]R flush_numbers[0..12]: [2, 1, 3, 1, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155433721Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[0]R generation[0..12]: [2, 1, 3, 1, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155452227Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155470914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[1]R flush_numbers[0..12]: [2, 1, 3, 1, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155484731Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[1]R generation[0..12]: [2, 1, 3, 1, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155497949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155511476Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[2]R flush_numbers[0..12]: [2, 1, 3, 1, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155524664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[2]R generation[0..12]: [2, 1, 3, 1, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155537561Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155551049Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:12:41.155563636Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155581093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155593861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155607158Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"":"downstairs","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155620016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155631534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"0","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155644311Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"1","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155657949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"client":"2","":"downstairs","session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155673206Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155685194Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"91bfb1bd-24c0-4754-b9e5-8a64d2eb1f03 is now active with session: 1158eeaa-fc3e-4562-bec8-2a3b7901f612","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.155697761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:41.15570935Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206,"session_id":"1158eeaa-fc3e-4562-bec8-2a3b7901f612"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-07T18:12:41.155729996Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 254 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Read block 440 len 4096 03/30 Read block 395 len 20480 04/30 Read block 159 len 28672 05/30 Flush 06/30 Write block 541 len 40960 data: 2 2 2 2 2 2 2 2 2 2 07/30 Write block 10 len 40960 data: 3 3 3 3 3 3 3 2 2 2 08/30 Read block 138 len 40960 09/30 Flush 10/30 Write block 569 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Write block 455 len 8192 data: 3 3 12/30 Read block 579 len 24576 13/30 Read block 188 len 36864 14/30 Write block 405 len 24576 data: 2 2 2 2 2 2 15/30 Write block 9 len 4096 data: 2 16/30 Write block 573 len 16384 data: 3 3 3 3 17/30 Read block 300 len 12288 18/30 Flush 19/30 Write block 368 len 12288 data: 2 2 2 20/30 Read block 160 len 28672 21/30 Read block 416 len 12288 22/30 Write block 68 len 12288 data: 2 2 2 23/30 Write block 500 len 36864 data: 3 2 2 2 2 2 2 2 2 24/30 Read block 72 len 28672 25/30 Read block 104 len 32768 26/30 Write block 440 len 28672 data: 2 2 2 2 2 2 2 27/30 Write block 316 len 12288 data: 2 2 2 28/30 Read block 294 len 4096 29/30 Write block 411 len 40960 data: 2 2 2 2 2 2 2 3 3 3 30/30 Write block 251 len 16384 data: 2 2 2 3 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:41.17559784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 1 1 1 1 1 1 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 3 3 3 3 3 3 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 4 4 4 T T T 13 260-279 4 4 4 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 3 3 3 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 6 6 6 T T T 21 420-439 3 3 3 3 3 3 T T T 22 440-459 4 4 4 6 6 6 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 T T T 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 6 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 3 3 3 3 3 3 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 5 5 5 7 7 7 F F F 13 260-279 4 4 4 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 5 5 5 7 7 7 F F F 21 420-439 5 5 5 7 7 7 F F F 22 440-459 5 5 5 7 7 7 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 5 5 5 7 7 7 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.446376793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.492458089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.493845573Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.493897853Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.493915509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.493933456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214} {"msg":"Crucible ed51c133-34f9-4403-b43a-dfef4f51f34e has session id: c5338641-9543-491c-923e-17e4754bf485","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.4945071Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Upstairs opts: Upstairs UUID: ed51c133-34f9-4403-b43a-dfef4f51f34e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.494547203Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Crucible stats registered with UUID: ed51c133-34f9-4403-b43a-dfef4f51f34e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.494570678Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:55.494591004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495178486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495205951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214} {"msg":"ed51c133-34f9-4403-b43a-dfef4f51f34e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495269579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495286526Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"0","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495312041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"0","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495325649Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"1","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495345015Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"1","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495358502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.49537251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495385468Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495398885Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495421071Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"ds_connection connected from Ok(127.0.0.1:33764)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495611606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:55.495716056Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:55.4957532Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"ds_connection connected from Ok(127.0.0.1:47061)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495781844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:12:55.495811459Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.495831835Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.496322374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"1","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.496360977Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"0","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.497011137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"1","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.497150761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"0","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.497539359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:12:55.497720486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.497076735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"ds_connection connected from Ok(127.0.0.1:57145)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.497245533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.498288021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.498855046Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[0]R flush_numbers[0..12]: [6, 1, 3, 7, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499489029Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[0]R generation[0..12]: [4, 1, 3, 5, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499520812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499540289Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[1]R flush_numbers[0..12]: [6, 1, 3, 7, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499557456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[1]R generation[0..12]: [4, 1, 3, 5, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499574553Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.49959165Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[2]R flush_numbers[0..12]: [6, 1, 3, 7, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499616065Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[2]R generation[0..12]: [4, 1, 3, 5, 1, 2, 2, 3, 1, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499633392Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499650539Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Max found gen is 6","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-07T18:13:05.499674074Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499690301Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499706138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499721666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"":"downstairs","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499737692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.49975258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"0","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499768917Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"1","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499784974Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"client":"2","":"downstairs","session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499801281Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499816068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"ed51c133-34f9-4403-b43a-dfef4f51f34e is now active with session: c5338641-9543-491c-923e-17e4754bf485","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499831805Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499846412Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214,"session_id":"c5338641-9543-491c-923e-17e4754bf485"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:05.499861389Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 124 len 20480 data: 2 2 2 2 2 03/30 Read block 283 len 36864 04/30 Read block 284 len 4096 05/30 Read block 335 len 16384 06/30 Write block 227 len 24576 data: 2 2 2 2 2 2 07/30 Read block 197 len 8192 08/30 Read block 78 len 16384 09/30 Read block 72 len 32768 10/30 Read block 383 len 12288 11/30 Read block 540 len 32768 12/30 Read block 156 len 32768 13/30 Write block 245 len 32768 data: 2 2 2 2 2 2 3 3 14/30 Read block 358 len 4096 15/30 Write block 194 len 24576 data: 2 2 2 2 2 2 16/30 Read block 205 len 20480 17/30 Read block 24 len 12288 18/30 Read block 430 len 8192 19/30 Write block 567 len 28672 data: 2 2 3 3 3 3 4 20/30 Write block 158 len 20480 data: 2 2 2 2 2 21/30 Write block 472 len 28672 data: 2 2 2 2 2 2 2 22/30 Read block 295 len 20480 23/30 Write block 509 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Flush 25/30 Write block 167 len 8192 data: 2 2 26/30 Read block 160 len 12288 27/30 Read block 351 len 40960 28/30 Write block 344 len 28672 data: 2 2 2 2 2 2 2 29/30 Write block 424 len 36864 data: 3 3 3 3 2 2 2 2 2 30/30 Read block 518 len 4096 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:05.523168608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 6 6 6 9 9 9 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 6 6 6 9 9 9 T T T 9 180-199 6 6 6 9 9 9 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 4 4 4 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 1 1 1 1 1 1 T T T 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 5 5 5 7 7 7 F F F 21 420-439 5 5 5 7 7 7 T T T 22 440-459 5 5 5 7 7 7 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 6 6 6 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 9 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 6 6 6 9 9 9 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 7 7 7 10 10 10 F F F 9 180-199 6 6 6 9 9 9 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 4 4 4 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 3 3 3 3 3 3 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 5 5 5 7 7 7 F F F 21 420-439 7 7 7 10 10 10 F F F 22 440-459 5 5 5 7 7 7 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 6 6 6 9 9 9 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 6 6 6 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 10 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.769910597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.816829867Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.817886142Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.817930584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.817946051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.817960438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222} {"msg":"Crucible 9d334fca-684b-412b-8cde-44087c04dbd6 has session id: d15d0c25-62ce-4e89-a82b-1ebac8bed97f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.818474833Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Upstairs opts: Upstairs UUID: 9d334fca-684b-412b-8cde-44087c04dbd6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.818522165Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Crucible stats registered with UUID: 9d334fca-684b-412b-8cde-44087c04dbd6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.81854906Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:09.818570585Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.81913909Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819165606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222} {"msg":"9d334fca-684b-412b-8cde-44087c04dbd6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819239852Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819258588Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"0","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819294442Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"0","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819322926Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"1","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819347952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"1","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819369718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"2","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819392873Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"2","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819415359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819439355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819472499Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51165)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819702006Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56112)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819741909Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58359)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.819769584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.8202228Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"2","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.820259753Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"0","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.820501199Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"1","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.820984259Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"0","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821011864Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"2","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.82125167Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"1","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821567371Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821653605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[0]R flush_numbers[0..12]: [6, 1, 3, 7, 1, 2, 9, 9, 10, 9, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.82184608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[0]R generation[0..12]: [4, 1, 3, 5, 1, 2, 6, 6, 7, 6, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821866936Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821882463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[1]R flush_numbers[0..12]: [6, 1, 3, 7, 1, 2, 9, 9, 10, 9, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821900509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[1]R generation[0..12]: [4, 1, 3, 5, 1, 2, 6, 6, 7, 6, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821914337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821928235Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[2]R flush_numbers[0..12]: [6, 1, 3, 7, 1, 2, 9, 9, 10, 9, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821941312Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[2]R generation[0..12]: [4, 1, 3, 5, 1, 2, 6, 6, 7, 6, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.82195473Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821968727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:13:09.821981435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.821998502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822012869Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822025557Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"":"downstairs","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822038414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822050332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"0","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.82206331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"1","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822076127Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"client":"2","":"downstairs","session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822089155Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822103732Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"9d334fca-684b-412b-8cde-44087c04dbd6 is now active with session: d15d0c25-62ce-4e89-a82b-1ebac8bed97f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.8221165Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822128067Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222,"session_id":"d15d0c25-62ce-4e89-a82b-1ebac8bed97f"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:09.822140065Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 185 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 170 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Read block 134 len 32768 04/30 Write block 457 len 40960 data: 3 3 2 2 2 2 2 2 2 2 05/30 Read block 416 len 4096 06/30 Flush 07/30 Write block 80 len 12288 data: 2 2 2 08/30 Read block 577 len 12288 09/30 Read block 126 len 40960 10/30 Flush 11/30 Write block 446 len 4096 data: 3 12/30 Read block 268 len 32768 13/30 Read block 62 len 16384 14/30 Write block 404 len 16384 data: 2 3 3 3 15/30 Read block 141 len 16384 16/30 Read block 271 len 32768 17/30 Write block 253 len 12288 data: 3 4 3 18/30 Read block 370 len 40960 19/30 Write block 325 len 32768 data: 3 3 3 3 2 2 2 2 20/30 Flush 21/30 Write block 269 len 36864 data: 2 2 2 2 2 2 2 2 2 22/30 Read block 465 len 20480 23/30 Write block 512 len 20480 data: 3 3 3 3 3 24/30 Flush 25/30 Write block 486 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 259 len 36864 27/30 Read block 234 len 12288 28/30 Read block 83 len 4096 29/30 Read block 86 len 8192 30/30 Read block 561 len 4096 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:09.842894706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 8 8 8 12 12 12 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 6 6 6 9 9 9 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 8 8 8 11 11 11 F F F 9 180-199 8 8 8 11 11 11 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 8 8 8 13 13 13 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 7 7 7 10 10 10 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 11 11 11 F F F 24 480-499 2 2 2 2 2 2 T T T 25 500-519 8 8 8 14 14 14 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 6 6 6 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 8, Max flush: 14 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 8 8 8 12 12 12 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 6 6 6 9 9 9 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 8 8 8 11 11 11 F F F 9 180-199 8 8 8 11 11 11 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 8 8 8 13 13 13 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 7 7 7 10 10 10 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 11 11 11 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 6 6 6 9 9 9 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.078303392Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:14.124353934Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.124953033Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.157435134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.15853808Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.158584892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.158602038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.158620755Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230} {"msg":"Crucible b699d080-9bcc-4cbf-ade0-f3068952d7f1 has session id: 435ca473-479c-4985-aea6-1f42a3ba9c9b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159171563Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Upstairs opts: Upstairs UUID: b699d080-9bcc-4cbf-ade0-f3068952d7f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159215775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Crucible stats registered with UUID: b699d080-9bcc-4cbf-ade0-f3068952d7f1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.15924177Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:14.159263096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159684658Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159717402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230} {"msg":"b699d080-9bcc-4cbf-ade0-f3068952d7f1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159777681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159803866Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"0","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159834521Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"0","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159850787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"1","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159872354Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"1","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159886331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"2","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159900518Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"2","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159913506Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.159928823Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.15994337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59172)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.160078036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41440)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.160112709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54964)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.160137634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.160788994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"1","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.160817929Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"2","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.161006504Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"0","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.161527557Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"1","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.161571909Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"2","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.161628709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"0","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162209561Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162312852Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[0]R flush_numbers[0..12]: [6, 1, 3, 7, 12, 2, 9, 9, 11, 11, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162405555Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[0]R generation[0..12]: [4, 1, 3, 5, 8, 2, 6, 6, 8, 8, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162425521Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162440248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[1]R flush_numbers[0..12]: [6, 1, 3, 7, 12, 2, 9, 9, 11, 11, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162453756Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[1]R generation[0..12]: [4, 1, 3, 5, 8, 2, 6, 6, 8, 8, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162466974Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162480761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[2]R flush_numbers[0..12]: [6, 1, 3, 7, 12, 2, 9, 9, 11, 11, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162494029Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[2]R generation[0..12]: [4, 1, 3, 5, 8, 2, 6, 6, 8, 8, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162507096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162520514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162532911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162545539Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162557927Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162580713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"":"downstairs","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.16259352Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162605238Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"0","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162618136Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"1","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162630913Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"client":"2","":"downstairs","session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162643461Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162655149Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"b699d080-9bcc-4cbf-ade0-f3068952d7f1 is now active with session: 435ca473-479c-4985-aea6-1f42a3ba9c9b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162667546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162679344Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230,"session_id":"435ca473-479c-4985-aea6-1f42a3ba9c9b"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:14.162691152Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 441 len 24576 data: 3 3 3 3 3 4 02/30 Write block 8 len 24576 data: 2 3 4 4 4 4 03/30 Write block 465 len 36864 data: 3 3 2 2 2 2 2 3 3 04/30 Write block 182 len 24576 data: 2 2 2 3 3 3 05/30 Write block 320 len 4096 data: 4 06/30 Read block 112 len 32768 07/30 Write block 556 len 40960 data: 2 2 2 2 2 2 2 2 2 2 08/30 Flush 09/30 Write block 120 len 4096 data: 2 10/30 Read block 303 len 16384 11/30 Flush 12/30 Write block 538 len 32768 data: 2 2 2 3 3 3 3 3 13/30 Read block 334 len 40960 14/30 Flush 15/30 Write block 289 len 12288 data: 2 2 2 16/30 Write block 426 len 24576 data: 4 4 3 3 3 3 17/30 Write block 589 len 4096 data: 2 18/30 Write block 411 len 4096 data: 3 19/30 Read block 471 len 12288 20/30 Flush 21/30 Write block 19 len 8192 data: 3 2 22/30 Read block 18 len 8192 23/30 Flush 24/30 Write block 97 len 12288 data: 2 2 2 25/30 Read block 343 len 8192 26/30 Read block 112 len 20480 27/30 Write block 70 len 36864 data: 3 2 2 2 2 2 2 2 2 28/30 Read block 553 len 28672 29/30 Read block 217 len 32768 30/30 Write block 326 len 40960 data: 4 4 4 3 3 3 3 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:14.181816374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 20 20 20 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 5 5 5 7 7 7 T T T 4 080-099 8 8 8 12 12 12 T T T 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 17 17 17 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 8 8 8 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 10 10 10 19 19 19 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 10 10 10 16 16 16 T T T 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 10 10 10 19 19 19 F F F 21 420-439 10 10 10 19 19 19 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 10 10 10 16 16 16 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 10, Max flush: 20 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 20 20 20 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 17 17 17 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 8 8 8 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 3 3 3 3 3 3 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 10 10 10 19 19 19 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 11 11 11 21 21 21 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 10 10 10 19 19 19 F F F 21 420-439 10 10 10 19 19 19 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 10 10 10 16 16 16 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 11, Max flush: 21 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.459797935Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.506970359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.508004728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.508050659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.508065796Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.508080014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238} {"msg":"Crucible 3e6c8b55-18a2-4b8d-9d00-a5d87e660f61 has session id: ba67d1d5-0ded-49ee-bd30-3fe7faa55e15","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.508726484Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Upstairs opts: Upstairs UUID: 3e6c8b55-18a2-4b8d-9d00-a5d87e660f61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.508766057Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Crucible stats registered with UUID: 3e6c8b55-18a2-4b8d-9d00-a5d87e660f61","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.508789532Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:18.508810059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509331132Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509367416Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238} {"msg":"3e6c8b55-18a2-4b8d-9d00-a5d87e660f61 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509425315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509444191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"0","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509470676Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"0","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509485354Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"1","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509502211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"1","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509526036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"2","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509555051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"2","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509894978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.50993794Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509968734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"ds_connection connected from Ok(127.0.0.1:42151)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.509998239Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"ds_connection connected from Ok(127.0.0.1:49488)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.510023014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"ds_connection connected from Ok(127.0.0.1:65059)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.51004681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.510548697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"2","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.51058843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"1","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.510621424Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"0","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.511149116Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"2","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.511340491Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"1","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.511436103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"0","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.511745705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.511856785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[0]R flush_numbers[0..12]: [20, 20, 3, 21, 21, 2, 17, 9, 11, 16, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.511983752Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[0]R generation[0..12]: [10, 10, 3, 11, 11, 2, 10, 6, 8, 10, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512012956Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512032332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[1]R flush_numbers[0..12]: [20, 20, 3, 21, 21, 2, 17, 9, 11, 16, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.51204644Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[1]R generation[0..12]: [10, 10, 3, 11, 11, 2, 10, 6, 8, 10, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512059807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512073775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[2]R flush_numbers[0..12]: [20, 20, 3, 21, 21, 2, 17, 9, 11, 16, 3, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512087512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[2]R generation[0..12]: [10, 10, 3, 11, 11, 2, 10, 6, 8, 10, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.51210083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512118727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512138463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.51215133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512163838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512176326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"":"downstairs","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512188943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512200481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"0","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512213369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"1","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512226097Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"client":"2","":"downstairs","session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512238904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512250802Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"3e6c8b55-18a2-4b8d-9d00-a5d87e660f61 is now active with session: ba67d1d5-0ded-49ee-bd30-3fe7faa55e15","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.5122634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512275218Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238,"session_id":"ba67d1d5-0ded-49ee-bd30-3fe7faa55e15"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:18.512287075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 216 len 16384 data: 2 2 2 2 02/30 Write block 241 len 36864 data: 2 2 2 2 3 3 3 3 3 03/30 Write block 542 len 16384 data: 4 4 4 4 04/30 Read block 535 len 28672 05/30 Flush 06/30 Write block 314 len 24576 data: 2 2 3 3 3 2 07/30 Read block 375 len 40960 08/30 Read block 149 len 8192 09/30 Read block 451 len 8192 10/30 Read block 130 len 40960 11/30 Read block 251 len 32768 12/30 Write block 135 len 20480 data: 2 2 2 3 3 13/30 Read block 391 len 12288 14/30 Read block 548 len 40960 15/30 Read block 224 len 4096 16/30 Read block 141 len 28672 17/30 Read block 219 len 36864 18/30 Read block 438 len 40960 19/30 Flush 20/30 Write block 552 len 40960 data: 2 2 2 2 3 3 3 3 3 3 21/30 Flush 22/30 Flush 23/30 Write block 389 len 4096 data: 2 24/30 Read block 383 len 40960 25/30 Write block 486 len 32768 data: 3 3 3 3 3 3 3 2 26/30 Read block 299 len 28672 27/30 Read block 70 len 4096 28/30 Read block 413 len 12288 29/30 Read block 271 len 40960 30/30 Write block 276 len 20480 data: 3 3 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:18.533408438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 20 20 20 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 12 12 12 23 23 23 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 8 8 8 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 12 12 12 22 22 22 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 12 12 12 22 22 22 F F F 13 260-279 8 8 8 14 14 14 T T T 14 280-299 10 10 10 19 19 19 T T T 15 300-319 12 12 12 23 23 23 F F F 16 320-339 11 11 11 21 21 21 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 10 10 10 19 19 19 F F F 21 420-439 10 10 10 19 19 19 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 9 9 9 15 15 15 T T T 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 12 12 12 24 24 24 F F F 28 560-579 12 12 12 24 24 24 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 12, Max flush: 24 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 20 20 20 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 11 11 11 21 21 21 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 12 12 12 23 23 23 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 8 8 8 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 12 12 12 22 22 22 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 12 12 12 22 22 22 F F F 13 260-279 13 13 13 25 25 25 F F F 14 280-299 13 13 13 25 25 25 F F F 15 300-319 12 12 12 23 23 23 F F F 16 320-339 11 11 11 21 21 21 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 10 10 10 19 19 19 F F F 21 420-439 10 10 10 19 19 19 F F F 22 440-459 10 10 10 16 16 16 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 13 13 13 25 25 25 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 12 12 12 24 24 24 F F F 28 560-579 12 12 12 24 24 24 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 13, Max flush: 25 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.791773357Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:22.838004845Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.838758736Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.872235123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.87338716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.873425223Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.873458527Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.873473234Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246} {"msg":"Crucible 002803ba-9474-47fc-967a-b925023a13e9 has session id: 117a6cb3-4196-4599-9c03-e40d71cc9e34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874168535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Upstairs opts: Upstairs UUID: 002803ba-9474-47fc-967a-b925023a13e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874213787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Crucible stats registered with UUID: 002803ba-9474-47fc-967a-b925023a13e9","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874233103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:22.874248971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874613433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874636719Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246} {"msg":"002803ba-9474-47fc-967a-b925023a13e9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874708255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874725932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"0","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874752027Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"0","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874768284Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"1","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.87479379Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"1","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874818865Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"2","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874842011Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"2","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874861417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874883513Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874904789Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"ds_connection connected from Ok(127.0.0.1:50454)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.874979376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"ds_connection connected from Ok(127.0.0.1:61793)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.875012059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"ds_connection connected from Ok(127.0.0.1:36767)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.875038645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.875844415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"1","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.87587359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"2","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.875897386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"0","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.876473479Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"1","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.876607434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"0","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.876769584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"2","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877146135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877259604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[0]R flush_numbers[0..12]: [20, 20, 3, 21, 21, 2, 23, 9, 11, 16, 22, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877407936Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[0]R generation[0..12]: [10, 10, 3, 11, 11, 2, 12, 6, 8, 10, 12, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877434861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877450288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[1]R flush_numbers[0..12]: [20, 20, 3, 21, 21, 2, 23, 9, 11, 16, 22, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877468145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[1]R generation[0..12]: [10, 10, 3, 11, 11, 2, 12, 6, 8, 10, 12, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877481793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.87749552Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[2]R flush_numbers[0..12]: [20, 20, 3, 21, 21, 2, 23, 9, 11, 16, 22, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877509037Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[2]R generation[0..12]: [10, 10, 3, 11, 11, 2, 12, 6, 8, 10, 12, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877522355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877535952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.87754851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877561328Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Next flush: 26","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-03-07T18:13:22.877586903Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877612018Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"":"downstairs","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877637354Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877660449Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"0","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877686305Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"1","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.87771288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"client":"2","":"downstairs","session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877739905Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877770249Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"002803ba-9474-47fc-967a-b925023a13e9 is now active with session: 117a6cb3-4196-4599-9c03-e40d71cc9e34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877791775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877811632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246,"session_id":"117a6cb3-4196-4599-9c03-e40d71cc9e34"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:22.877834687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 32768 data: 3 2 2 2 2 3 3 3 02/30 Write block 567 len 8192 data: 3 3 03/30 Read block 85 len 24576 04/30 Read block 66 len 12288 05/30 Write block 233 len 36864 data: 2 2 2 2 2 2 2 2 3 06/30 Read block 416 len 36864 07/30 Read block 110 len 40960 08/30 Flush 09/30 Write block 2 len 4096 data: 2 10/30 Read block 54 len 4096 11/30 Write block 419 len 12288 data: 4 4 3 12/30 Read block 333 len 28672 13/30 Read block 317 len 4096 14/30 Write block 87 len 32768 data: 2 2 2 2 2 2 2 2 15/30 Flush 16/30 Write block 306 len 20480 data: 2 2 2 2 2 17/30 Read block 68 len 40960 18/30 Read block 260 len 24576 19/30 Write block 86 len 8192 data: 2 3 20/30 Write block 482 len 28672 data: 2 2 2 2 4 4 4 21/30 Read block 61 len 32768 22/30 Read block 211 len 40960 23/30 Flush 24/30 Write block 438 len 40960 data: 2 2 3 4 4 4 4 4 5 2 25/30 Write block 85 len 20480 data: 2 3 4 3 3 26/30 Read block 297 len 40960 27/30 Read block 378 len 24576 28/30 Read block 296 len 40960 29/30 Read block 455 len 20480 30/30 Read block 94 len 32768 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:22.899826179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 14 14 14 28 28 28 T T T 5 100-119 2 2 2 2 2 2 F F F 6 120-139 12 12 12 23 23 23 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 12 12 12 22 22 22 F F F 11 220-239 14 14 14 26 26 26 F F F 12 240-259 14 14 14 26 26 26 F F F 13 260-279 13 13 13 25 25 25 F F F 14 280-299 13 13 13 25 25 25 F F F 15 300-319 14 14 14 28 28 28 F F F 16 320-339 11 11 11 21 21 21 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 14 14 14 27 27 27 T T T 22 440-459 10 10 10 16 16 16 T T T 23 460-479 10 10 10 16 16 16 F F F 24 480-499 14 14 14 28 28 28 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 12 12 12 24 24 24 F F F 28 560-579 14 14 14 26 26 26 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 14, Max flush: 28 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 15 15 15 29 29 29 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 12 12 12 23 23 23 F F F 7 140-159 6 6 6 9 9 9 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 12 12 12 22 22 22 F F F 11 220-239 14 14 14 26 26 26 F F F 12 240-259 14 14 14 26 26 26 F F F 13 260-279 13 13 13 25 25 25 F F F 14 280-299 13 13 13 25 25 25 F F F 15 300-319 14 14 14 28 28 28 F F F 16 320-339 11 11 11 21 21 21 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 10 10 10 16 16 16 F F F 24 480-499 14 14 14 28 28 28 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 12 12 12 24 24 24 F F F 28 560-579 14 14 14 26 26 26 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 15, Max flush: 29 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.173852241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.221420282Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.222235391Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.222288431Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.222313937Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.222332893Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254} {"msg":"Crucible 54f0e198-14b0-4016-95bf-8c1af76028e1 has session id: fa043ca2-29c1-4cfa-98bb-dc17edcbab30","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.222853157Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Upstairs opts: Upstairs UUID: 54f0e198-14b0-4016-95bf-8c1af76028e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.222976994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Crucible stats registered with UUID: 54f0e198-14b0-4016-95bf-8c1af76028e1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223039153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:27.223062379Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223461625Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223495318Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254} {"msg":"54f0e198-14b0-4016-95bf-8c1af76028e1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223547019Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223561876Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"0","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.22359464Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"0","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223615566Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"1","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223631093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"1","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223644271Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"2","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223669716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"2","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223695811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223721226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223744382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"ds_connection connected from Ok(127.0.0.1:52690)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223773867Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"ds_connection connected from Ok(127.0.0.1:56528)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223841734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"ds_connection connected from Ok(127.0.0.1:65182)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.223875928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.224559442Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"1","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.224712523Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"0","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.225140104Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"1","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.22526821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"0","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.225445727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"2","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.225737484Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.225829376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226096177Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"2","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[0]R flush_numbers[0..12]: [27, 20, 3, 21, 29, 2, 23, 9, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226661293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[0]R generation[0..12]: [14, 10, 3, 11, 15, 2, 12, 6, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226683978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226698646Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[1]R flush_numbers[0..12]: [27, 20, 3, 21, 29, 2, 23, 9, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226712273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[1]R generation[0..12]: [14, 10, 3, 11, 15, 2, 12, 6, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226725731Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226739408Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[2]R flush_numbers[0..12]: [27, 20, 3, 21, 29, 2, 23, 9, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226752696Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[2]R generation[0..12]: [14, 10, 3, 11, 15, 2, 12, 6, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226765833Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226779451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226792209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-07T18:13:27.226811815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226824912Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.2268373Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"":"downstairs","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226850348Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226865985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"0","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226879632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"1","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.22689283Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"client":"2","":"downstairs","session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226906187Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226918465Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"54f0e198-14b0-4016-95bf-8c1af76028e1 is now active with session: fa043ca2-29c1-4cfa-98bb-dc17edcbab30","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226931133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226942911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254,"session_id":"fa043ca2-29c1-4cfa-98bb-dc17edcbab30"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:27.226954778Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 4096 data: 2 02/30 Read block 375 len 16384 03/30 Read block 169 len 16384 04/30 Write block 177 len 24576 data: 4 3 3 3 3 4 05/30 Read block 480 len 36864 06/30 Read block 259 len 40960 07/30 Read block 330 len 20480 08/30 Write block 440 len 8192 data: 4 5 09/30 Write block 276 len 40960 data: 4 4 3 3 3 2 2 2 2 2 10/30 Write block 542 len 4096 data: 5 11/30 Flush 12/30 Write block 488 len 40960 data: 5 4 4 4 4 3 2 2 2 3 13/30 Read block 487 len 32768 14/30 Flush 15/30 Write block 154 len 32768 data: 2 2 2 2 3 3 3 3 16/30 Read block 71 len 24576 17/30 Write block 228 len 24576 data: 3 3 3 3 3 3 18/30 Flush 19/30 Write block 210 len 20480 data: 2 2 2 2 2 20/30 Write block 145 len 36864 data: 3 3 3 3 2 2 2 2 2 21/30 Write block 178 len 12288 data: 4 4 4 22/30 Read block 555 len 32768 23/30 Read block 211 len 8192 24/30 Read block 433 len 20480 25/30 Write block 333 len 8192 data: 3 3 26/30 Read block 303 len 16384 27/30 Write block 295 len 12288 data: 2 2 2 28/30 Read block 310 len 8192 29/30 Write block 453 len 32768 data: 3 3 4 4 4 4 3 3 30/30 Read block 30 len 4096 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 5 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:27.248048806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 15 15 15 29 29 29 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 12 12 12 23 23 23 F F F 7 140-159 16 16 16 32 32 32 T T T 8 160-179 16 16 16 32 32 32 T T T 9 180-199 16 16 16 30 30 30 T T T 10 200-219 12 12 12 22 22 22 T T T 11 220-239 16 16 16 32 32 32 F F F 12 240-259 14 14 14 26 26 26 F F F 13 260-279 16 16 16 30 30 30 F F F 14 280-299 16 16 16 30 30 30 T T T 15 300-319 14 14 14 28 28 28 F F F 16 320-339 11 11 11 21 21 21 T T T 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 16 16 16 30 30 30 T T T 23 460-479 10 10 10 16 16 16 T T T 24 480-499 16 16 16 31 31 31 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 14 14 14 26 26 26 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 16, Max flush: 32 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 15 15 15 29 29 29 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 12 12 12 23 23 23 F F F 7 140-159 17 17 17 33 33 33 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 17 17 17 33 33 33 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 16 16 16 32 32 32 F F F 12 240-259 14 14 14 26 26 26 F F F 13 260-279 16 16 16 30 30 30 F F F 14 280-299 17 17 17 33 33 33 F F F 15 300-319 14 14 14 28 28 28 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 17 17 17 33 33 33 F F F 23 460-479 17 17 17 33 33 33 F F F 24 480-499 16 16 16 31 31 31 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 14 14 14 26 26 26 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 17, Max flush: 33 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.531846271Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.579181775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.580253986Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.580312436Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.580331912Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.580350679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262} {"msg":"Crucible 1aa53134-ca92-414e-9394-b050e0c3980b has session id: ee2a8529-0262-4017-80db-729dcaca3cb0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.580968244Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Upstairs opts: Upstairs UUID: 1aa53134-ca92-414e-9394-b050e0c3980b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581012236Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Crucible stats registered with UUID: 1aa53134-ca92-414e-9394-b050e0c3980b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581031543Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:41.58104484Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581563584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581589619Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262} {"msg":"1aa53134-ca92-414e-9394-b050e0c3980b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581653827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581683222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"0","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581712637Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"0","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581726874Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"1","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581740822Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"1","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581755469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"2","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581780114Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"2","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.58180478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581831105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.58185674Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35040)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581880056Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58193)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.58196558Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:57208)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.581999974Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.582697345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"0","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.582737977Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"1","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.583439337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"1","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.583481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"0","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.583939135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"2","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.584149136Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.584176431Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.584509819Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"2","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[0]R flush_numbers[0..12]: [30, 20, 3, 21, 29, 2, 23, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585029593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[0]R generation[0..12]: [16, 10, 3, 11, 15, 2, 12, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585050319Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585065376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[1]R flush_numbers[0..12]: [30, 20, 3, 21, 29, 2, 23, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585079274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[1]R generation[0..12]: [16, 10, 3, 11, 15, 2, 12, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585092851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585106509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[2]R flush_numbers[0..12]: [30, 20, 3, 21, 29, 2, 23, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585119726Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[2]R generation[0..12]: [16, 10, 3, 11, 15, 2, 12, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585132674Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585150981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Max found gen is 18","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585170607Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585183714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585196182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.58520885Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"":"downstairs","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585221698Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585233575Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"0","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585246813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"1","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.58525966Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"client":"2","":"downstairs","session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585272528Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585284466Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"1aa53134-ca92-414e-9394-b050e0c3980b is now active with session: ee2a8529-0262-4017-80db-729dcaca3cb0","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:13:41.585297124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585309311Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262,"session_id":"ee2a8529-0262-4017-80db-729dcaca3cb0"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:41.585321169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 94 len 40960 data: 3 2 2 3 3 3 2 2 2 2 02/30 Read block 415 len 16384 03/30 Read block 190 len 28672 04/30 Write block 403 len 20480 data: 2 3 4 4 4 05/30 Write block 579 len 16384 data: 2 2 2 2 06/30 Read block 471 len 8192 07/30 Read block 187 len 4096 08/30 Flush 09/30 Write block 258 len 40960 data: 3 3 3 3 2 2 2 2 2 2 10/30 Flush 11/30 Write block 129 len 8192 data: 2 2 12/30 Read block 535 len 28672 13/30 Read block 400 len 32768 14/30 Read block 513 len 16384 15/30 Write block 305 len 8192 data: 2 3 16/30 Read block 28 len 12288 17/30 Write block 579 len 24576 data: 3 3 3 3 2 2 18/30 Write block 90 len 24576 data: 3 3 3 3 4 3 19/30 Read block 168 len 4096 20/30 Read block 13 len 32768 21/30 Read block 488 len 4096 22/30 Write block 492 len 32768 data: 5 4 3 3 3 4 3 3 23/30 Read block 410 len 40960 24/30 Read block 157 len 12288 25/30 Read block 530 len 24576 26/30 Read block 386 len 40960 27/30 Write block 371 len 24576 data: 2 2 2 2 2 2 28/30 Write block 290 len 36864 data: 3 3 2 2 2 3 3 3 2 29/30 Write block 471 len 20480 data: 3 4 4 3 3 30/30 Read block 459 len 12288 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 2 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Sent false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 3 5 Done 20 18 17 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:41.607034043Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 18 18 18 34 34 34 T T T 5 100-119 18 18 18 34 34 34 F F F 6 120-139 12 12 12 23 23 23 T T T 7 140-159 17 17 17 33 33 33 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 17 17 17 33 33 33 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 16 16 16 32 32 32 F F F 12 240-259 18 18 18 35 35 35 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 17 17 17 33 33 33 T T T 15 300-319 14 14 14 28 28 28 T T T 16 320-339 17 17 17 33 33 33 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 5 5 5 7 7 7 T T T 19 380-399 13 13 13 25 25 25 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 17 17 17 33 33 33 F F F 23 460-479 17 17 17 33 33 33 T T T 24 480-499 16 16 16 31 31 31 T T T 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 18 18 18 34 34 34 T T T 29 580-599 18 18 18 34 34 34 T T T Max gen: 18, Max flush: 35 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 10 10 10 20 20 20 F F F 2 040-059 3 3 3 3 3 3 F F F 3 060-079 11 11 11 21 21 21 F F F 4 080-099 19 19 19 36 36 36 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 19 19 19 36 36 36 F F F 7 140-159 17 17 17 33 33 33 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 17 17 17 33 33 33 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 16 16 16 32 32 32 F F F 12 240-259 18 18 18 35 35 35 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 19 19 19 36 36 36 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 19 19 19 36 36 36 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 18 18 18 34 34 34 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 17 17 17 33 33 33 F F F 23 460-479 19 19 19 36 36 36 F F F 24 480-499 19 19 19 36 36 36 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 19 19 19 36 36 36 F F F Max gen: 19, Max flush: 36 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.897143459Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.943896651Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.944917262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.944963113Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.944978491Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.944992908Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270} {"msg":"Crucible 689798b4-02db-41c2-9343-3abf0d0c4e71 has session id: cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.945596106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Upstairs opts: Upstairs UUID: 689798b4-02db-41c2-9343-3abf0d0c4e71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.945638159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Crucible stats registered with UUID: 689798b4-02db-41c2-9343-3abf0d0c4e71","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.945661664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:45.94568244Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946168201Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946189877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270} {"msg":"689798b4-02db-41c2-9343-3abf0d0c4e71 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.94627977Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946311964Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"0","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946340189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"0","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946355216Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946369733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946382681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"2","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946396458Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"2","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946409316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946422534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.94644174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49425)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946752382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:45.946809862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:45.946840176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:45.946864292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946882838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"ds_connection connected from Ok(127.0.0.1:60456)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.946916342Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.947479648Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"2","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.947508083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"0","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.948139696Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"0","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.94827776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"2","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.948739525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:45.94882036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.947581769Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62873)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.94774288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.948961564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.949682191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[0]R flush_numbers[0..12]: [30, 20, 3, 21, 36, 34, 36, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950347248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[0]R generation[0..12]: [16, 10, 3, 11, 19, 18, 19, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950371593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950387541Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[1]R flush_numbers[0..12]: [30, 20, 3, 21, 36, 34, 36, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950401438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[1]R generation[0..12]: [16, 10, 3, 11, 19, 18, 19, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950414885Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950428633Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[2]R flush_numbers[0..12]: [30, 20, 3, 21, 36, 34, 36, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.95044192Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[2]R generation[0..12]: [16, 10, 3, 11, 19, 18, 19, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950455188Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950468775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950481333Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950494091Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Next flush: 37","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-07T18:13:55.950521496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950534933Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"":"downstairs","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950547671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950559219Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"0","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950572126Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"1","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950584954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"client":"2","":"downstairs","session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950599131Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950611239Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"689798b4-02db-41c2-9343-3abf0d0c4e71 is now active with session: cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950623927Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950635535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270,"session_id":"cb0c7bd5-52e1-4522-be62-f4cfaf5e8ca8"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:13:55.950647362Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 357 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 0 len 12288 03/30 Flush 04/30 Write block 524 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Read block 398 len 24576 06/30 Read block 15 len 12288 07/30 Read block 69 len 40960 08/30 Write block 15 len 28672 data: 4 4 3 3 4 3 2 09/30 Read block 136 len 28672 10/30 Read block 65 len 8192 11/30 Read block 214 len 20480 12/30 Write block 419 len 24576 data: 5 5 4 3 3 4 13/30 Flush 14/30 Write block 433 len 40960 data: 2 2 2 2 2 3 3 5 6 5 15/30 Write block 474 len 28672 data: 4 4 3 3 3 2 2 16/30 Write block 79 len 28672 data: 2 3 3 3 2 2 3 17/30 Read block 462 len 28672 18/30 Read block 71 len 4096 19/30 Write block 440 len 4096 data: 6 20/30 Read block 143 len 32768 21/30 Write block 42 len 20480 data: 2 2 2 2 2 22/30 Read block 507 len 32768 23/30 Read block 114 len 28672 24/30 Read block 451 len 12288 25/30 Read block 104 len 8192 26/30 Flush 27/30 Write block 586 len 20480 data: 2 2 2 3 2 28/30 Read block 35 len 24576 29/30 Read block 413 len 36864 30/30 Read block 300 len 24576 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:13:55.974265533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 38 38 38 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 39 39 39 F F F 3 060-079 20 20 20 39 39 39 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 19 19 19 36 36 36 F F F 7 140-159 17 17 17 33 33 33 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 17 17 17 33 33 33 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 16 16 16 32 32 32 F F F 12 240-259 18 18 18 35 35 35 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 19 19 19 36 36 36 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 20 20 20 37 37 37 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 20 20 20 39 39 39 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 19 19 19 36 36 36 T T T Max gen: 20, Max flush: 39 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 38 38 38 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 20 20 20 39 39 39 F F F 3 060-079 20 20 20 39 39 39 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 19 19 19 36 36 36 F F F 7 140-159 17 17 17 33 33 33 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 17 17 17 33 33 33 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 16 16 16 32 32 32 F F F 12 240-259 18 18 18 35 35 35 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 19 19 19 36 36 36 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 20 20 20 37 37 37 F F F 19 380-399 13 13 13 25 25 25 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 20 20 20 39 39 39 F F F 22 440-459 20 20 20 39 39 39 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 21 21 21 40 40 40 F F F Max gen: 21, Max flush: 40 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.204061697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.24826406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.248885605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.248927728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.248942925Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.248957552Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278} {"msg":"Crucible cbd9897f-f529-4ebc-b6f8-916795a0c71e has session id: 6344aa2c-11d6-4fde-ba50-30717bc27fdd","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.249500362Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Upstairs opts: Upstairs UUID: cbd9897f-f529-4ebc-b6f8-916795a0c71e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.249534016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Crucible stats registered with UUID: cbd9897f-f529-4ebc-b6f8-916795a0c71e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.249549863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:00.24956295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250014577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250040322Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278} {"msg":"cbd9897f-f529-4ebc-b6f8-916795a0c71e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250110639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250128895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"0","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250154491Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"0","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250168148Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250182276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250196683Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"2","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250221298Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"2","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250246624Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250271969Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250298884Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41046)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250340836Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52450)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250560766Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:00.250609997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:00.250630563Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:00.250655658Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.250681843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.251103545Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"0","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.251283932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"2","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.251649124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"0","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.251785429Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"2","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.252198613Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:00.252313681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.252056139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:63974)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.25226716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.253365137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254004569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[0]R flush_numbers[0..12]: [38, 38, 39, 39, 39, 34, 36, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254694381Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[0]R generation[0..12]: [20, 20, 20, 20, 20, 18, 19, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254716947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254732314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[1]R flush_numbers[0..12]: [38, 38, 39, 39, 39, 34, 36, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.25475242Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[1]R generation[0..12]: [20, 20, 20, 20, 20, 18, 19, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254766708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254780635Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[2]R flush_numbers[0..12]: [38, 38, 39, 39, 39, 34, 36, 33, 33, 33, 33, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254794123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[2]R generation[0..12]: [20, 20, 20, 20, 20, 18, 19, 17, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.25480751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254821467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254834265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254847423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.25486002Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254872678Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"":"downstairs","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254885596Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254897273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"0","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254910551Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"1","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254931847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"client":"2","":"downstairs","session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254948934Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254961492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"cbd9897f-f529-4ebc-b6f8-916795a0c71e is now active with session: 6344aa2c-11d6-4fde-ba50-30717bc27fdd","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254974159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.254985837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278,"session_id":"6344aa2c-11d6-4fde-ba50-30717bc27fdd"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:10.255002514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 443 len 8192 data: 5 5 02/30 Write block 366 len 12288 data: 3 2 3 03/30 Read block 580 len 40960 04/30 Read block 333 len 28672 05/30 Write block 40 len 8192 data: 3 2 06/30 Read block 150 len 28672 07/30 Write block 565 len 24576 data: 3 2 4 4 4 4 08/30 Write block 312 len 20480 data: 2 2 3 3 4 09/30 Write block 594 len 8192 data: 2 2 10/30 Read block 496 len 40960 11/30 Read block 305 len 24576 12/30 Read block 175 len 32768 13/30 Write block 180 len 16384 data: 5 4 5 4 14/30 Flush 15/30 Write block 389 len 28672 data: 3 2 2 2 2 2 2 16/30 Read block 533 len 24576 17/30 Write block 235 len 24576 data: 3 3 3 3 3 3 18/30 Write block 450 len 12288 data: 2 2 2 19/30 Read block 18 len 4096 20/30 Write block 194 len 24576 data: 3 3 3 3 3 3 21/30 Write block 429 len 4096 data: 4 22/30 Read block 558 len 28672 23/30 Read block 465 len 24576 24/30 Write block 246 len 36864 data: 4 4 4 4 3 4 4 4 5 25/30 Read block 456 len 16384 26/30 Flush 27/30 Write block 136 len 8192 data: 3 3 28/30 Read block 448 len 16384 29/30 Read block 157 len 20480 30/30 Read block 89 len 24576 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:10.275649875Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 38 38 38 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 22 22 22 41 41 41 F F F 3 060-079 20 20 20 39 39 39 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 19 19 19 36 36 36 T T T 7 140-159 17 17 17 33 33 33 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 22 22 22 42 42 42 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 22 22 22 41 41 41 F F F 19 380-399 22 22 22 42 42 42 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 22 22 22 42 42 42 F F F 22 440-459 22 22 22 42 42 42 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 22 22 22 41 41 41 F F F 29 580-599 22 22 22 41 41 41 F F F Max gen: 22, Max flush: 42 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 38 38 38 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 22 22 22 41 41 41 F F F 3 060-079 20 20 20 39 39 39 F F F 4 080-099 20 20 20 39 39 39 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 23 23 23 43 43 43 F F F 7 140-159 17 17 17 33 33 33 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 22 22 22 42 42 42 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 22 22 22 41 41 41 F F F 19 380-399 22 22 22 42 42 42 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 22 22 22 42 42 42 F F F 22 440-459 22 22 22 42 42 42 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 8 8 8 14 14 14 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 22 22 22 41 41 41 F F F 29 580-599 22 22 22 41 41 41 F F F Max gen: 23, Max flush: 43 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.51127636Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:14.555053272Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.555719659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.588399164Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.58955526Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.589600271Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.589615489Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.589629896Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286} {"msg":"Crucible 2a1081a9-111e-4fcd-bcc4-0b56c3fefc91 has session id: 8329e205-43e4-44e3-9b32-2301cbd1338b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.590343094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Upstairs opts: Upstairs UUID: 2a1081a9-111e-4fcd-bcc4-0b56c3fefc91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.590445575Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Crucible stats registered with UUID: 2a1081a9-111e-4fcd-bcc4-0b56c3fefc91","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.590467591Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:14.590480519Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.590975027Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591000292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286} {"msg":"2a1081a9-111e-4fcd-bcc4-0b56c3fefc91 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591061601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591088086Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"0","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591114582Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"0","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591129209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"1","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591152774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"1","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.5911772Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"2","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591199346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"2","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591220962Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591245927Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591270483Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40948)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.59128511Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63634)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591585135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62900)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.591634805Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.592118156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"2","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.592351842Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"1","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.59252848Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"0","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.592931206Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"2","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593117791Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"1","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593374893Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"0","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593537563Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593594013Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[0]R flush_numbers[0..12]: [38, 38, 41, 39, 39, 34, 43, 33, 33, 42, 33, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593919093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[0]R generation[0..12]: [20, 20, 22, 20, 20, 18, 23, 17, 17, 22, 17, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593940569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593956176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[1]R flush_numbers[0..12]: [38, 38, 41, 39, 39, 34, 43, 33, 33, 42, 33, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593969813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[1]R generation[0..12]: [20, 20, 22, 20, 20, 18, 23, 17, 17, 22, 17, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593983231Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.593996918Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[2]R flush_numbers[0..12]: [38, 38, 41, 39, 39, 34, 43, 33, 33, 42, 33, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594010166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[2]R generation[0..12]: [20, 20, 22, 20, 20, 18, 23, 17, 17, 22, 17, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594023653Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594037691Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594050408Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594063166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594075584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338bWait for a query_work_queue command to finish before sending IO "} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594127854Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"":"downstairs","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.59415176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594165387Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"0","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594179475Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"1","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594192642Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"client":"2","":"downstairs","session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.59420583Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594218107Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"2a1081a9-111e-4fcd-bcc4-0b56c3fefc91 is now active with session: 8329e205-43e4-44e3-9b32-2301cbd1338b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594230835Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.594242823Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286,"session_id":"8329e205-43e4-44e3-9b32-2301cbd1338b"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:14.59425483Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 386 len 36864 data: 2 2 2 4 3 3 3 3 3 03/30 Read block 229 len 36864 04/30 Read block 40 len 4096 05/30 Read block 280 len 36864 06/30 Read block 567 len 40960 07/30 Read block 363 len 20480 08/30 Read block 21 len 12288 09/30 Read block 271 len 24576 10/30 Read block 249 len 4096 11/30 Write block 247 len 8192 data: 5 5 12/30 Write block 515 len 8192 data: 4 4 13/30 Read block 127 len 36864 14/30 Flush 15/30 Write block 143 len 40960 data: 2 2 4 4 4 4 3 3 3 3 16/30 Flush 17/30 Write block 84 len 28672 data: 3 4 4 5 4 4 4 18/30 Write block 85 len 4096 data: 5 19/30 Read block 423 len 8192 20/30 Flush 21/30 Flush 22/30 Write block 564 len 12288 data: 3 4 3 23/30 Write block 9 len 28672 data: 4 5 5 5 5 4 5 24/30 Read block 221 len 36864 25/30 Flush 26/30 Write block 510 len 40960 data: 3 3 4 4 4 5 5 3 3 2 27/30 Read block 226 len 28672 28/30 Write block 68 len 32768 data: 3 3 4 3 3 3 3 3 29/30 Write block 87 len 12288 data: 6 5 5 30/30 Read block 0 len 4096 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:14.615426664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 49 49 49 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 22 22 22 41 41 41 F F F 3 060-079 20 20 20 39 39 39 T T T 4 080-099 24 24 24 47 47 47 T T T 5 100-119 18 18 18 34 34 34 F F F 6 120-139 23 23 23 43 43 43 F F F 7 140-159 24 24 24 46 46 46 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 24 24 24 45 45 45 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 22 22 22 41 41 41 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 22 22 22 42 42 42 F F F 22 440-459 22 22 22 42 42 42 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 24 24 24 45 45 45 T T T 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 24 24 24 49 49 49 F F F 29 580-599 22 22 22 41 41 41 F F F Max gen: 24, Max flush: 49 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 49 49 49 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 22 22 22 41 41 41 F F F 3 060-079 25 25 25 50 50 50 F F F 4 080-099 25 25 25 50 50 50 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 23 23 23 43 43 43 F F F 7 140-159 24 24 24 46 46 46 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 17 17 17 33 33 33 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 24 24 24 45 45 45 F F F 13 260-279 18 18 18 35 35 35 F F F 14 280-299 19 19 19 36 36 36 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 17 17 17 33 33 33 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 22 22 22 41 41 41 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 20 20 20 38 38 38 F F F 21 420-439 22 22 22 42 42 42 F F F 22 440-459 22 22 22 42 42 42 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 24 24 24 49 49 49 F F F 29 580-599 22 22 22 41 41 41 F F F Max gen: 25, Max flush: 50 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.896352319Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:18.939810001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.940220935Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.971556118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.972162236Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.97219564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.972209467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.972223255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294} {"msg":"Crucible 9d729ba3-b307-4498-a4a1-4239c488854b has session id: 5c77f8a2-e27e-4613-a24f-5371d9f8e4ec","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.972764215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Upstairs opts: Upstairs UUID: 9d729ba3-b307-4498-a4a1-4239c488854b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.972805047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Crucible stats registered with UUID: 9d729ba3-b307-4498-a4a1-4239c488854b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.972830092Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:18.972851748Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973217271Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973242136Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294} {"msg":"9d729ba3-b307-4498-a4a1-4239c488854b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973308054Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973326701Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"0","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973352776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"0","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973366433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"1","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973380391Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"1","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973393418Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"2","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973407056Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"2","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973420003Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.97343528Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973457156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:53174)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973496309Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:32771)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973525834Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:39697)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.973557178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.974242911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"1","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.974278764Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"2","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.974535167Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"0","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.974996702Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"1","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975026306Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"2","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975042363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"0","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975612588Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975728886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[0]R flush_numbers[0..12]: [49, 38, 41, 50, 50, 34, 43, 46, 33, 42, 33, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975791755Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[0]R generation[0..12]: [24, 20, 22, 25, 25, 18, 23, 24, 17, 22, 17, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975812431Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975827028Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[1]R flush_numbers[0..12]: [49, 38, 41, 50, 50, 34, 43, 46, 33, 42, 33, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975840746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[1]R generation[0..12]: [24, 20, 22, 25, 25, 18, 23, 24, 17, 22, 17, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975853953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975867681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[2]R flush_numbers[0..12]: [49, 38, 41, 50, 50, 34, 43, 46, 33, 42, 33, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975880738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[2]R generation[0..12]: [24, 20, 22, 25, 25, 18, 23, 24, 17, 22, 17, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975898735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975913492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.97592622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975939217Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975951675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:14:18.975964093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"":"downstairs","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.975984929Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.976000746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"0","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.976024022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"1","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.976038359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"client":"2","":"downstairs","session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.976052556Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.976064664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"9d729ba3-b307-4498-a4a1-4239c488854b is now active with session: 5c77f8a2-e27e-4613-a24f-5371d9f8e4ec","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.976077142Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.97608888Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294,"session_id":"5c77f8a2-e27e-4613-a24f-5371d9f8e4ec"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:18.976100827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 339 len 24576 data: 2 2 2 2 2 3 02/30 Write block 401 len 20480 data: 2 2 3 4 5 03/30 Write block 216 len 36864 data: 3 3 3 3 2 2 2 2 2 04/30 Write block 405 len 36864 data: 6 5 5 3 3 3 4 3 3 05/30 Read block 555 len 32768 06/30 Write block 246 len 4096 data: 5 07/30 Flush 08/30 Write block 578 len 36864 data: 2 4 4 4 4 3 3 2 3 09/30 Read block 480 len 24576 10/30 Read block 487 len 16384 11/30 Write block 447 len 36864 data: 3 2 2 3 3 3 4 4 5 12/30 Write block 347 len 40960 data: 3 3 3 3 2 2 2 2 2 2 13/30 Read block 348 len 16384 14/30 Write block 92 len 8192 data: 4 4 15/30 Write block 468 len 32768 data: 3 3 3 4 5 5 5 5 16/30 Write block 290 len 8192 data: 4 4 17/30 Read block 111 len 24576 18/30 Flush 19/30 Write block 69 len 32768 data: 4 5 4 4 4 4 4 3 20/30 Read block 431 len 8192 21/30 Flush 22/30 Write block 232 len 32768 data: 4 4 3 4 4 4 4 4 23/30 Write block 47 len 8192 data: 2 2 24/30 Read block 359 len 4096 25/30 Read block 418 len 12288 26/30 Read block 389 len 28672 27/30 Write block 82 len 12288 data: 4 3 4 28/30 Write block 272 len 12288 data: 3 3 3 29/30 Read block 468 len 40960 30/30 Read block 459 len 4096 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:18.995704161Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 49 49 49 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 22 22 22 41 41 41 T T T 3 060-079 26 26 26 53 53 53 F F F 4 080-099 26 26 26 52 52 52 T T T 5 100-119 18 18 18 34 34 34 F F F 6 120-139 23 23 23 43 43 43 F F F 7 140-159 24 24 24 46 46 46 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 26 26 26 51 51 51 F F F 11 220-239 26 26 26 51 51 51 T T T 12 240-259 26 26 26 51 51 51 F F F 13 260-279 18 18 18 35 35 35 T T T 14 280-299 26 26 26 52 52 52 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 26 26 26 52 52 52 F F F 18 360-379 22 22 22 41 41 41 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 26 26 26 51 51 51 F F F 21 420-439 22 22 22 42 42 42 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 26 26 26 52 52 52 F F F Max gen: 26, Max flush: 53 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 49 49 49 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 27 27 27 54 54 54 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 27 27 27 54 54 54 F F F 5 100-119 18 18 18 34 34 34 F F F 6 120-139 23 23 23 43 43 43 F F F 7 140-159 24 24 24 46 46 46 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 26 26 26 51 51 51 F F F 11 220-239 27 27 27 54 54 54 F F F 12 240-259 26 26 26 51 51 51 F F F 13 260-279 27 27 27 54 54 54 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 26 26 26 52 52 52 F F F 18 360-379 22 22 22 41 41 41 F F F 19 380-399 24 24 24 45 45 45 F F F 20 400-419 26 26 26 51 51 51 F F F 21 420-439 22 22 22 42 42 42 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 26 26 26 52 52 52 F F F Max gen: 27, Max flush: 54 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.276868054Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.323435139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.324374946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.324422137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.324438514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.324453141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302} {"msg":"Crucible 780507d5-86cd-4f87-9e56-39c4f18d29ec has session id: e286f20c-5f46-4f90-89ae-717a30b811c4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.324965596Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Upstairs opts: Upstairs UUID: 780507d5-86cd-4f87-9e56-39c4f18d29ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325135395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Crucible stats registered with UUID: 780507d5-86cd-4f87-9e56-39c4f18d29ec","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325166769Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:23.325187995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325704629Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325737973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302} {"msg":"780507d5-86cd-4f87-9e56-39c4f18d29ec active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.32581288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325848003Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"0","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325894734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"0","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325929428Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"1","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325961882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"1","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.325989757Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"2","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326020761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"2","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326049976Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.32607904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326166304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50267)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326203107Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:59348)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326237001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35263)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326389803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326865925Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"1","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.326995691Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"0","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.327100931Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"2","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.327438999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"1","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.327583112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"0","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.327915231Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"2","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328152627Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328234472Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[0]R flush_numbers[0..12]: [49, 38, 54, 53, 54, 34, 43, 46, 33, 42, 51, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328454951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[0]R generation[0..12]: [24, 20, 27, 26, 27, 18, 23, 24, 17, 22, 26, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328474047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328493724Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[1]R flush_numbers[0..12]: [49, 38, 54, 53, 54, 34, 43, 46, 33, 42, 51, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.32851128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[1]R generation[0..12]: [24, 20, 27, 26, 27, 18, 23, 24, 17, 22, 26, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328535536Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328562471Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[2]R flush_numbers[0..12]: [49, 38, 54, 53, 54, 34, 43, 46, 33, 42, 51, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328583617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[2]R generation[0..12]: [24, 20, 27, 26, 27, 18, 23, 24, 17, 22, 26, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328599514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:14:23.328624979Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328648555Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.32867374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328689168Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328702895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"":"downstairs","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328716572Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328732959Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"0","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328756125Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"1","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328780141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"client":"2","":"downstairs","session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328803056Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328825032Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"780507d5-86cd-4f87-9e56-39c4f18d29ec is now active with session: e286f20c-5f46-4f90-89ae-717a30b811c4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328848648Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328870044Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302,"session_id":"e286f20c-5f46-4f90-89ae-717a30b811c4"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:23.328883101Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 24576 data: 3 2 2 2 2 2 02/30 Flush 03/30 Write block 272 len 4096 data: 4 04/30 Write block 371 len 8192 data: 3 3 05/30 Write block 162 len 20480 data: 3 2 2 2 2 06/30 Read block 76 len 16384 07/30 Write block 108 len 12288 data: 3 3 3 08/30 Read block 217 len 40960 09/30 Read block 15 len 28672 10/30 Write block 356 len 20480 data: 3 3 3 3 3 11/30 Write block 414 len 20480 data: 3 3 3 3 4 12/30 Write block 424 len 8192 data: 5 4 13/30 Write block 363 len 20480 data: 3 3 3 4 3 14/30 Read block 211 len 12288 15/30 Read block 236 len 28672 16/30 Read block 297 len 4096 17/30 Write block 11 len 24576 data: 6 6 6 5 6 5 18/30 Write block 207 len 4096 data: 2 19/30 Write block 116 len 28672 data: 2 2 2 2 3 2 2 20/30 Write block 51 len 28672 data: 2 2 2 2 2 2 2 21/30 Read block 307 len 40960 22/30 Read block 248 len 28672 23/30 Write block 399 len 8192 data: 2 2 24/30 Flush 25/30 Write block 542 len 28672 data: 6 5 5 5 3 3 3 26/30 Read block 253 len 16384 27/30 Write block 373 len 16384 data: 3 3 3 3 28/30 Read block 442 len 20480 29/30 Read block 301 len 32768 30/30 Read block 308 len 40960 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:23.349068488Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 28 28 28 56 56 56 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 27 27 27 54 54 54 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 28 28 28 56 56 56 F F F 7 140-159 24 24 24 46 46 46 F F F 8 160-179 28 28 28 56 56 56 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 28 28 28 56 56 56 F F F 11 220-239 27 27 27 54 54 54 F F F 12 240-259 26 26 26 51 51 51 F F F 13 260-279 28 28 28 56 56 56 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 28 28 28 56 56 56 T T T 19 380-399 28 28 28 56 56 56 F F F 20 400-419 28 28 28 56 56 56 F F F 21 420-439 28 28 28 56 56 56 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 16 16 16 30 30 30 T T T 28 560-579 26 26 26 52 52 52 F F F 29 580-599 26 26 26 52 52 52 F F F Max gen: 28, Max flush: 56 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 20 20 20 38 38 38 F F F 2 040-059 28 28 28 56 56 56 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 27 27 27 54 54 54 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 28 28 28 56 56 56 F F F 7 140-159 24 24 24 46 46 46 F F F 8 160-179 28 28 28 56 56 56 F F F 9 180-199 22 22 22 42 42 42 F F F 10 200-219 28 28 28 56 56 56 F F F 11 220-239 27 27 27 54 54 54 F F F 12 240-259 26 26 26 51 51 51 F F F 13 260-279 28 28 28 56 56 56 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 22 22 22 41 41 41 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 28 28 28 56 56 56 F F F 20 400-419 28 28 28 56 56 56 F F F 21 420-439 28 28 28 56 56 56 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 20 20 20 38 38 38 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 26 26 26 52 52 52 F F F Max gen: 29, Max flush: 57 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.585136461Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.630403208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.631313599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.63136577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.631383766Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.631401553Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310} {"msg":"Crucible 523a59b9-a53d-46e7-bd7a-ef6a127f1809 has session id: 4193cc02-62de-4783-b5b6-f6529d78f5e6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.63195743Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Upstairs opts: Upstairs UUID: 523a59b9-a53d-46e7-bd7a-ef6a127f1809, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632003492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Crucible stats registered with UUID: 523a59b9-a53d-46e7-bd7a-ef6a127f1809","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632027397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:27.632042234Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.6325022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632523716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310} {"msg":"523a59b9-a53d-46e7-bd7a-ef6a127f1809 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632622277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632642123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"0","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632678297Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"0","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632704582Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"1","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632729258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"1","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632760182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632786647Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632810363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632835738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.632858643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:27.63287638Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:27.632892047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:27.632907405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57025)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.633070104Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.633115216Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42853)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.633194401Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.633825405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"0","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.633858229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"1","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.634422554Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"1","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.634538323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"0","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.634973302Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:27.635071394Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.634323623Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39948)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.63450226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.635727863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.636532364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[0]R flush_numbers[0..12]: [56, 38, 56, 53, 54, 56, 56, 46, 56, 42, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637159768Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[0]R generation[0..12]: [28, 20, 28, 26, 27, 28, 28, 24, 28, 22, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637185843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.63720128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[1]R flush_numbers[0..12]: [56, 38, 56, 53, 54, 56, 56, 46, 56, 42, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637215548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[1]R generation[0..12]: [28, 20, 28, 26, 27, 28, 28, 24, 28, 22, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637230715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637244692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[2]R flush_numbers[0..12]: [56, 38, 56, 53, 54, 56, 56, 46, 56, 42, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.63725838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[2]R generation[0..12]: [28, 20, 28, 26, 27, 28, 28, 24, 28, 22, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637277656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637291644Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637304241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637317159Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637338325Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637351043Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"":"downstairs","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637363961Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637375948Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"0","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637389026Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"1","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637402053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"client":"2","":"downstairs","session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637415091Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637427079Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"523a59b9-a53d-46e7-bd7a-ef6a127f1809 is now active with session: 4193cc02-62de-4783-b5b6-f6529d78f5e6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637439536Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637451134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310,"session_id":"4193cc02-62de-4783-b5b6-f6529d78f5e6"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:37.637463032Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 40960 data: 3 3 3 3 3 5 5 5 6 5 02/30 Read block 306 len 40960 03/30 Write block 392 len 8192 data: 4 4 04/30 Write block 184 len 16384 data: 4 4 4 4 05/30 Write block 312 len 8192 data: 3 3 06/30 Read block 481 len 12288 07/30 Write block 214 len 28672 data: 3 2 4 4 4 4 3 08/30 Write block 37 len 16384 data: 2 2 2 4 09/30 Flush 10/30 Write block 399 len 36864 data: 3 3 3 3 4 5 7 6 6 11/30 Flush 12/30 Write block 255 len 36864 data: 4 3 3 4 4 4 4 3 3 13/30 Write block 525 len 16384 data: 3 3 3 3 14/30 Read block 316 len 8192 15/30 Flush 16/30 Write block 129 len 4096 data: 3 17/30 Read block 144 len 28672 18/30 Write block 392 len 12288 data: 5 5 4 19/30 Read block 44 len 12288 20/30 Flush 21/30 Write block 206 len 28672 data: 2 3 2 2 3 3 3 22/30 Read block 529 len 8192 23/30 Write block 198 len 20480 data: 4 4 2 2 3 24/30 Write block 153 len 24576 data: 3 3 3 3 3 4 25/30 Read block 164 len 16384 26/30 Read block 320 len 20480 27/30 Read block 518 len 20480 28/30 Read block 478 len 32768 29/30 Write block 29 len 8192 data: 2 2 30/30 Read block 151 len 12288 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:37.658535454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 30 30 30 58 58 58 T T T 2 040-059 30 30 30 58 58 58 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 27 27 27 54 54 54 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 30 30 30 61 61 61 F F F 7 140-159 24 24 24 46 46 46 T T T 8 160-179 30 30 30 58 58 58 F F F 9 180-199 30 30 30 58 58 58 T T T 10 200-219 30 30 30 58 58 58 T T T 11 220-239 30 30 30 58 58 58 F F F 12 240-259 30 30 30 60 60 60 F F F 13 260-279 30 30 30 60 60 60 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 30 30 30 58 58 58 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 30 30 30 61 61 61 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 28 28 28 56 56 56 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 30 30 30 60 60 60 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 26 26 26 52 52 52 F F F Max gen: 30, Max flush: 61 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 31 31 31 62 62 62 F F F 2 040-059 30 30 30 58 58 58 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 27 27 27 54 54 54 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 30 30 30 61 61 61 F F F 7 140-159 31 31 31 62 62 62 F F F 8 160-179 30 30 30 58 58 58 F F F 9 180-199 31 31 31 62 62 62 F F F 10 200-219 31 31 31 62 62 62 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 30 30 30 60 60 60 F F F 13 260-279 30 30 30 60 60 60 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 30 30 30 58 58 58 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 30 30 30 61 61 61 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 28 28 28 56 56 56 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 30 30 30 60 60 60 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 26 26 26 52 52 52 F F F Max gen: 31, Max flush: 62 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.917308027Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.965275314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.966345396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.966389668Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.966404495Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.966418823Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318} {"msg":"Crucible f3f83eb8-c37e-4ac8-94db-103416248195 has session id: 78c03c8f-adfd-4d45-ace2-0418f95818b4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.96719087Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Upstairs opts: Upstairs UUID: f3f83eb8-c37e-4ac8-94db-103416248195, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.967252918Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Crucible stats registered with UUID: f3f83eb8-c37e-4ac8-94db-103416248195","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.967283363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:41.967307628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.967830382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.967855807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318} {"msg":"f3f83eb8-c37e-4ac8-94db-103416248195 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.967957708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.967975415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"0","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.96800081Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"0","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968014897Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"1","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968035324Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"1","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968058669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"2","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968084975Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"2","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.9681097Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968134685Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.96816458Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63332)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968221659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51964)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968359374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35360)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.968407525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.969063144Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"0","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.969095038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"1","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.969778151Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"0","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.969904368Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"1","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.970312423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"2","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.970390878Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.970423322Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.970823108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"2","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[0]R flush_numbers[0..12]: [56, 62, 58, 53, 54, 56, 61, 62, 58, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971347761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[0]R generation[0..12]: [28, 31, 30, 26, 27, 28, 30, 31, 30, 31, 31, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971372307Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971387484Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[1]R flush_numbers[0..12]: [56, 62, 58, 53, 54, 56, 61, 62, 58, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.97140754Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[1]R generation[0..12]: [28, 31, 30, 26, 27, 28, 30, 31, 30, 31, 31, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971421927Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971435765Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[2]R flush_numbers[0..12]: [56, 62, 58, 53, 54, 56, 61, 62, 58, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971449032Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[2]R generation[0..12]: [28, 31, 30, 26, 27, 28, 30, 31, 30, 31, 31, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.97146229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971486086Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.97151345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971544935Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971574839Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971604204Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"":"downstairs","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971634128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971662183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"0","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971690038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"1","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971723262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"client":"2","":"downstairs","session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971758215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971775932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"f3f83eb8-c37e-4ac8-94db-103416248195 is now active with session: 78c03c8f-adfd-4d45-ace2-0418f95818b4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971791809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971806386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318,"session_id":"78c03c8f-adfd-4d45-ace2-0418f95818b4"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:41.971821313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 8192 data: 3 3 02/30 Read block 273 len 36864 03/30 Read block 150 len 36864 04/30 Write block 211 len 4096 data: 4 05/30 Read block 307 len 40960 06/30 Write block 446 len 8192 data: 6 4 07/30 Flush 08/30 Write block 32 len 12288 data: 2 2 2 09/30 Read block 90 len 24576 10/30 Read block 263 len 4096 11/30 Write block 311 len 12288 data: 2 4 4 12/30 Read block 440 len 32768 13/30 Read block 346 len 32768 14/30 Read block 402 len 36864 15/30 Flush 16/30 Write block 255 len 36864 data: 5 4 4 5 5 5 5 4 4 17/30 Read block 431 len 40960 18/30 Flush 19/30 Flush 20/30 Write block 580 len 36864 data: 5 5 5 4 4 3 4 3 3 21/30 Write block 422 len 8192 data: 4 4 22/30 Write block 581 len 40960 data: 6 6 5 5 4 5 4 4 4 3 23/30 Read block 176 len 20480 24/30 Write block 144 len 36864 data: 3 5 5 5 5 4 4 4 4 25/30 Write block 93 len 20480 data: 5 5 4 3 4 26/30 Read block 124 len 24576 27/30 Read block 175 len 12288 28/30 Write block 591 len 32768 data: 2 2 2 3 3 2 2 2 29/30 Write block 423 len 32768 data: 5 6 5 5 5 4 5 4 30/30 Read block 10 len 36864 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:41.993018392Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 32 32 32 64 64 64 F F F 2 040-059 30 30 30 58 58 58 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 27 27 27 54 54 54 T T T 5 100-119 28 28 28 56 56 56 F F F 6 120-139 30 30 30 61 61 61 F F F 7 140-159 31 31 31 62 62 62 T T T 8 160-179 30 30 30 58 58 58 F F F 9 180-199 31 31 31 62 62 62 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 32 32 32 65 65 65 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 32 32 32 64 64 64 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 30 30 30 61 61 61 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 28 28 28 56 56 56 T T T 22 440-459 32 32 32 63 63 63 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 32 32 32 63 63 63 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 30 30 30 60 60 60 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 26 26 26 52 52 52 T T T Max gen: 32, Max flush: 65 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 32 32 32 64 64 64 F F F 2 040-059 30 30 30 58 58 58 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 28 28 28 56 56 56 F F F 6 120-139 30 30 30 61 61 61 F F F 7 140-159 33 33 33 66 66 66 F F F 8 160-179 30 30 30 58 58 58 F F F 9 180-199 31 31 31 62 62 62 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 32 32 32 65 65 65 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 32 32 32 64 64 64 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 29 29 29 57 57 57 F F F 19 380-399 30 30 30 61 61 61 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 33 33 33 66 66 66 F F F 22 440-459 32 32 32 63 63 63 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 32 32 32 63 63 63 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 30 30 30 60 60 60 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 F F F Max gen: 33, Max flush: 66 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.250741141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.29723586Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.298198212Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.298247253Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.29826179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.298276267Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326} {"msg":"Crucible 6aad0ed1-8b69-40ca-bc45-3a91e6bfbc28 has session id: 01e30135-c483-4f04-b241-f4de3daff15f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.298834774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Upstairs opts: Upstairs UUID: 6aad0ed1-8b69-40ca-bc45-3a91e6bfbc28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.298875426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Crucible stats registered with UUID: 6aad0ed1-8b69-40ca-bc45-3a91e6bfbc28","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.298898502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:46.298919038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.29939941Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299433014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326} {"msg":"6aad0ed1-8b69-40ca-bc45-3a91e6bfbc28 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299489973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299515818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"0","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299545153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"0","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.29955978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"1","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299573987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"1","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299596183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"2","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299623218Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"2","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.29972335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299761273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299790697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64420)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299818202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59319)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299847647Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63113)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.299877511Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.300581191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"0","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.300609566Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"1","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.301314565Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"1","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.301355038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"0","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.301923623Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.302072625Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.302108859Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"2","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.302689211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"2","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[0]R flush_numbers[0..12]: [56, 64, 58, 53, 66, 56, 61, 66, 58, 62, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303269634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[0]R generation[0..12]: [28, 32, 30, 26, 33, 28, 30, 33, 30, 31, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.30329014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303304797Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[1]R flush_numbers[0..12]: [56, 64, 58, 53, 66, 56, 61, 66, 58, 62, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303318335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[1]R generation[0..12]: [28, 32, 30, 26, 33, 28, 30, 33, 30, 31, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303331762Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.30334599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[2]R flush_numbers[0..12]: [56, 64, 58, 53, 66, 56, 61, 66, 58, 62, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303359297Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[2]R generation[0..12]: [28, 32, 30, 26, 33, 28, 30, 33, 30, 31, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303372605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303386042Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.30339846Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303411158Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Next flush: 67","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:14:46.303423595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303446241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"":"downstairs","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303459538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303471336Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"0","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303484304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"1","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303497012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"client":"2","":"downstairs","session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303509749Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303521657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"6aad0ed1-8b69-40ca-bc45-3a91e6bfbc28 is now active with session: 01e30135-c483-4f04-b241-f4de3daff15f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303534155Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303545933Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326,"session_id":"01e30135-c483-4f04-b241-f4de3daff15f"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:46.303557711Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 339 len 32768 data: 3 3 3 3 3 4 3 3 02/30 Write block 317 len 20480 data: 4 4 3 5 4 03/30 Flush 04/30 Write block 289 len 20480 data: 3 5 5 3 3 05/30 Flush 06/30 Write block 355 len 16384 data: 3 4 4 4 07/30 Write block 286 len 24576 data: 2 2 2 4 6 6 08/30 Write block 439 len 24576 data: 4 7 7 6 6 6 09/30 Write block 187 len 32768 data: 5 3 3 3 3 3 2 4 10/30 Read block 573 len 8192 11/30 Read block 405 len 4096 12/30 Write block 372 len 28672 data: 4 4 4 4 4 2 2 13/30 Write block 293 len 12288 data: 4 3 4 14/30 Write block 37 len 8192 data: 3 3 15/30 Write block 375 len 36864 data: 5 5 3 3 2 2 2 2 2 16/30 Write block 486 len 8192 data: 5 5 17/30 Write block 104 len 20480 data: 3 3 2 2 4 18/30 Read block 168 len 28672 19/30 Write block 271 len 12288 data: 3 5 4 20/30 Read block 434 len 28672 21/30 Read block 206 len 8192 22/30 Read block 447 len 36864 23/30 Write block 366 len 8192 data: 5 4 24/30 Read block 427 len 20480 25/30 Flush 26/30 Write block 179 len 32768 data: 6 7 6 6 5 5 5 5 27/30 Read block 342 len 4096 28/30 Write block 372 len 12288 data: 5 5 5 29/30 Write block 183 len 16384 data: 6 6 6 6 30/30 Write block 139 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 3 3 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:46.321597413Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 34 34 34 69 69 69 F F F 2 040-059 30 30 30 58 58 58 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 34 34 34 69 69 69 F F F 6 120-139 30 30 30 61 61 61 T T T 7 140-159 33 33 33 66 66 66 F F F 8 160-179 30 30 30 58 58 58 T T T 9 180-199 34 34 34 69 69 69 T T T 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 34 34 34 69 69 69 F F F 14 280-299 34 34 34 69 69 69 F F F 15 300-319 34 34 34 67 67 67 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 34 34 34 69 69 69 F F F 18 360-379 34 34 34 69 69 69 T T T 19 380-399 34 34 34 69 69 69 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 30 30 30 60 60 60 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 F F F Max gen: 34, Max flush: 69 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 F F F 1 020-039 34 34 34 69 69 69 F F F 2 040-059 30 30 30 58 58 58 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 34 34 34 69 69 69 F F F 6 120-139 35 35 35 70 70 70 F F F 7 140-159 33 33 33 66 66 66 F F F 8 160-179 35 35 35 70 70 70 F F F 9 180-199 35 35 35 70 70 70 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 34 34 34 69 69 69 F F F 14 280-299 34 34 34 69 69 69 F F F 15 300-319 34 34 34 67 67 67 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 34 34 34 69 69 69 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 34 34 34 69 69 69 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 30 30 30 60 60 60 F F F 27 540-559 29 29 29 57 57 57 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 F F F Max gen: 35, Max flush: 70 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.573474693Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:50.617123208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.617675756Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.649947017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.650855219Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.650892182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.650905929Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.650920147Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334} {"msg":"Crucible 6f71ae1d-9690-4fef-b39d-c3db804e7808 has session id: 5b87439e-0fd4-4e45-b405-409455fa75c5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.651543741Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Upstairs opts: Upstairs UUID: 6f71ae1d-9690-4fef-b39d-c3db804e7808, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.651593452Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Crucible stats registered with UUID: 6f71ae1d-9690-4fef-b39d-c3db804e7808","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.651623986Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:50.651651501Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652155928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652178684Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334} {"msg":"6f71ae1d-9690-4fef-b39d-c3db804e7808 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652245411Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652264188Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"0","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652290843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"0","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652304531Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"1","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652318608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"1","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652331746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"2","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652345263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"2","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.65236108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652380897Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652394994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:49412)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652510533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48444)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.652551835Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61452)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.6525798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.653294417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"1","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.653328921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"2","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.65354851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"0","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.654184703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"1","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.654226125Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"2","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.654258159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"0","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.654831333Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.654946822Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[0]R flush_numbers[0..12]: [56, 69, 58, 53, 66, 69, 70, 66, 70, 70, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655022278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[0]R generation[0..12]: [28, 34, 30, 26, 33, 34, 35, 33, 35, 35, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655042864Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655057431Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[1]R flush_numbers[0..12]: [56, 69, 58, 53, 66, 69, 70, 66, 70, 70, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655071449Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[1]R generation[0..12]: [28, 34, 30, 26, 33, 34, 35, 33, 35, 35, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655085456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655099284Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[2]R flush_numbers[0..12]: [56, 69, 58, 53, 66, 69, 70, 66, 70, 70, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.65511774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[2]R generation[0..12]: [28, 34, 30, 26, 33, 34, 35, 33, 35, 35, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655131658Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655145175Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655157663Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.65517039Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655187827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-07T18:14:50.655208163Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"":"downstairs","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655237138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655260384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"0","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.65528004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"1","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655293448Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"client":"2","":"downstairs","session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655306305Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655327032Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"6f71ae1d-9690-4fef-b39d-c3db804e7808 is now active with session: 5b87439e-0fd4-4e45-b405-409455fa75c5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655351057Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.655373943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334,"session_id":"5b87439e-0fd4-4e45-b405-409455fa75c5"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:50.65538894Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 120 len 4096 data: 4 02/30 Write block 192 len 8192 data: 4 3 03/30 Read block 276 len 4096 04/30 Read block 334 len 20480 05/30 Write block 44 len 12288 data: 3 3 3 06/30 Write block 537 len 40960 data: 2 3 3 3 4 7 6 6 6 4 07/30 Write block 311 len 4096 data: 3 08/30 Read block 487 len 24576 09/30 Read block 196 len 16384 10/30 Flush 11/30 Write block 113 len 32768 data: 3 3 3 3 3 3 3 5 12/30 Read block 14 len 24576 13/30 Write block 293 len 24576 data: 5 4 5 4 4 3 14/30 Read block 258 len 24576 15/30 Read block 260 len 12288 16/30 Write block 9 len 12288 data: 5 6 7 17/30 Write block 106 len 20480 data: 3 3 5 4 4 18/30 Write block 45 len 24576 data: 4 4 3 3 2 2 19/30 Write block 265 len 16384 data: 3 3 3 2 20/30 Write block 297 len 4096 data: 5 21/30 Write block 119 len 32768 data: 4 6 3 3 2 3 3 3 22/30 Write block 151 len 16384 data: 5 5 4 4 23/30 Read block 525 len 20480 24/30 Read block 188 len 40960 25/30 Read block 301 len 40960 26/30 Read block 375 len 8192 27/30 Write block 49 len 36864 data: 3 3 3 3 3 3 3 3 3 28/30 Read block 395 len 32768 29/30 Write block 124 len 20480 data: 4 4 4 3 3 30/30 Read block 64 len 4096 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 8 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:50.674979156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 56 56 56 T T T 1 020-039 34 34 34 69 69 69 F F F 2 040-059 36 36 36 71 71 71 T T T 3 060-079 26 26 26 53 53 53 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 34 34 34 69 69 69 T T T 6 120-139 36 36 36 71 71 71 T T T 7 140-159 33 33 33 66 66 66 T T T 8 160-179 35 35 35 70 70 70 F F F 9 180-199 36 36 36 71 71 71 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 34 34 34 69 69 69 T T T 14 280-299 34 34 34 69 69 69 T T T 15 300-319 36 36 36 71 71 71 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 34 34 34 69 69 69 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 34 34 34 69 69 69 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 36 36 36 71 71 71 F F F 27 540-559 36 36 36 71 71 71 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 F F F Max gen: 36, Max flush: 71 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 72 72 72 F F F 1 020-039 34 34 34 69 69 69 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 26 26 26 53 53 53 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 37 37 37 72 72 72 F F F 6 120-139 37 37 37 72 72 72 F F F 7 140-159 37 37 37 72 72 72 F F F 8 160-179 35 35 35 70 70 70 F F F 9 180-199 36 36 36 71 71 71 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 37 37 37 72 72 72 F F F 14 280-299 37 37 37 72 72 72 F F F 15 300-319 36 36 36 71 71 71 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 34 34 34 69 69 69 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 34 34 34 69 69 69 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 36 36 36 71 71 71 F F F 27 540-559 36 36 36 71 71 71 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 F F F Max gen: 37, Max flush: 72 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:54.975045671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:55.020984204Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.021723467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.055102493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.056193221Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.056247771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.056265867Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.056283644Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342} {"msg":"Crucible b4ee0c23-b4ba-4a35-bae4-beddf3b246b1 has session id: 6f657a29-b8d4-4811-a442-951d08623b25","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.056909968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Upstairs opts: Upstairs UUID: b4ee0c23-b4ba-4a35-bae4-beddf3b246b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.05695323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Crucible stats registered with UUID: b4ee0c23-b4ba-4a35-bae4-beddf3b246b1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.056978195Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:55.057022098Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057444479Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057471434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342} {"msg":"b4ee0c23-b4ba-4a35-bae4-beddf3b246b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057529364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057554799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"0","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057581124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"0","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057596261Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"1","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057611738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"1","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057630045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"2","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057644692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"2","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.05765782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057672577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057693443Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"ds_connection connected from Ok(127.0.0.1:63335)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057713939Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"ds_connection connected from Ok(127.0.0.1:61183)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057791615Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"ds_connection connected from Ok(127.0.0.1:45252)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.057901905Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.058480018Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"1","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.058509642Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"2","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.058607004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"0","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059131767Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"2","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059165571Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"1","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059192686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"0","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059776818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059803313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[0]R flush_numbers[0..12]: [72, 69, 72, 53, 66, 72, 72, 72, 70, 71, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059836047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[0]R generation[0..12]: [37, 34, 37, 26, 33, 37, 37, 37, 35, 36, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059853434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059867981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[1]R flush_numbers[0..12]: [72, 69, 72, 53, 66, 72, 72, 72, 70, 71, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059881738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[1]R generation[0..12]: [37, 34, 37, 26, 33, 37, 37, 37, 35, 36, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059895296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059909163Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[2]R flush_numbers[0..12]: [72, 69, 72, 53, 66, 72, 72, 72, 70, 71, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059922701Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[2]R generation[0..12]: [37, 34, 37, 26, 33, 37, 37, 37, 35, 36, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059936138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059950866Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059963813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.059991428Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060014994Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060050347Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"":"downstairs","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060073503Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.0600875Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"0","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060103817Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"1","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060128083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"client":"2","":"downstairs","session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060150899Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060174824Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"b4ee0c23-b4ba-4a35-bae4-beddf3b246b1 is now active with session: 6f657a29-b8d4-4811-a442-951d08623b25","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060200819Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :30,"time":"2025-03-07T18:14:55.060229494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342,"session_id":"6f657a29-b8d4-4811-a442-951d08623b25"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:55.060254229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 151 len 32768 data: 6 6 5 5 4 4 4 5 02/30 Read block 525 len 16384 03/30 Read block 149 len 16384 04/30 Read block 416 len 28672 05/30 Flush 06/30 Flush 07/30 Write block 64 len 8192 data: 2 2 08/30 Write block 72 len 20480 data: 5 5 5 5 4 09/30 Read block 151 len 16384 10/30 Read block 47 len 32768 11/30 Read block 29 len 8192 12/30 Read block 412 len 40960 13/30 Read block 113 len 8192 14/30 Read block 0 len 8192 15/30 Read block 314 len 12288 16/30 Read block 590 len 36864 17/30 Read block 526 len 32768 18/30 Write block 307 len 40960 data: 3 3 3 3 4 5 5 4 4 5 19/30 Read block 417 len 36864 20/30 Read block 366 len 28672 21/30 Read block 162 len 20480 22/30 Read block 42 len 28672 23/30 Write block 148 len 4096 data: 6 24/30 Read block 365 len 20480 25/30 Read block 40 len 16384 26/30 Flush 27/30 Write block 453 len 4096 data: 5 28/30 Write block 22 len 16384 data: 2 2 2 2 29/30 Read block 104 len 8192 30/30 Write block 380 len 20480 data: 3 3 3 3 2 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:55.081382801Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 72 72 72 F F F 1 020-039 34 34 34 69 69 69 T T T 2 040-059 37 37 37 72 72 72 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 37 37 37 72 72 72 F F F 6 120-139 37 37 37 72 72 72 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 35 35 35 70 70 70 F F F 9 180-199 36 36 36 71 71 71 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 37 37 37 72 72 72 F F F 14 280-299 37 37 37 72 72 72 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 34 34 34 69 69 69 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 34 34 34 69 69 69 T T T 20 400-419 30 30 30 59 59 59 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 T T T 23 460-479 26 26 26 52 52 52 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 36 36 36 71 71 71 F F F 27 540-559 36 36 36 71 71 71 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 F F F Max gen: 38, Max flush: 75 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 72 72 72 F F F 1 020-039 39 39 39 76 76 76 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 37 37 37 72 72 72 F F F 6 120-139 37 37 37 72 72 72 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 35 35 35 70 70 70 F F F 9 180-199 36 36 36 71 71 71 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 37 37 37 72 72 72 F F F 14 280-299 37 37 37 72 72 72 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 34 34 34 69 69 69 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 39 39 39 76 76 76 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 39 39 39 76 76 76 F F F 23 460-479 26 26 26 52 52 52 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 36 36 36 71 71 71 F F F 27 540-559 36 36 36 71 71 71 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 F F F Max gen: 39, Max flush: 76 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.367890645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:59.413552068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.414319716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.447316153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.448435236Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.448476038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.448490325Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.448504633Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350} {"msg":"Crucible ab9acd42-0adc-4007-bed5-6b81342358a4 has session id: 4eac2028-b1f7-44da-b373-78f43f1c2d79","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.44916761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Upstairs opts: Upstairs UUID: ab9acd42-0adc-4007-bed5-6b81342358a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449197774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Crucible stats registered with UUID: ab9acd42-0adc-4007-bed5-6b81342358a4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449212152Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:59.44922462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449645542Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449670827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350} {"msg":"ab9acd42-0adc-4007-bed5-6b81342358a4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449741774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449759811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"0","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449786666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"0","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449801433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"1","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.44981708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"1","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449836836Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"2","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449863282Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"2","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.449958824Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.450028601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.450070713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"ds_connection connected from Ok(127.0.0.1:41734)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.450098818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"ds_connection connected from Ok(127.0.0.1:57153)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.450126133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"ds_connection connected from Ok(127.0.0.1:32964)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.450155597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.451034635Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"1","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.451091084Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"2","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.451605059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"0","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.451859782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"1","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452051257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"2","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452280814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"0","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452397713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452579889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[0]R flush_numbers[0..12]: [72, 76, 72, 75, 66, 72, 72, 75, 70, 71, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452817785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[0]R generation[0..12]: [37, 39, 37, 38, 33, 37, 37, 38, 35, 36, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452853248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452879953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[1]R flush_numbers[0..12]: [72, 76, 72, 75, 66, 72, 72, 75, 70, 71, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452906379Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[1]R generation[0..12]: [37, 39, 37, 38, 33, 37, 37, 38, 35, 36, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452931624Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452954979Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[2]R flush_numbers[0..12]: [72, 76, 72, 75, 66, 72, 72, 75, 70, 71, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.452980045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[2]R generation[0..12]: [37, 39, 37, 38, 33, 37, 37, 38, 35, 36, 32, 30]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-07T18:14:59.45300711Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453048232Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453074628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453098993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.45311691Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453139955Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"":"downstairs","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453164481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453183437Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"0","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453201194Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"client":"1","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.45322441Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 pid":1350,"client":"2","":"downstairs","session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453253754Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.45327604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"ab9acd42-0adc-4007-bed5-6b81342358a4 is now active with session: 4eac2028-b1f7-44da-b373-78f43f1c2d79","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453289888Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453301905Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350,"session_id":"4eac2028-b1f7-44da-b373-78f43f1c2d79"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:14:59.453314053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 28672 data: 5 5 7 7 6 6 5 02/30 Read block 404 len 20480 03/30 Read block 298 len 16384 04/30 Write block 356 len 12288 data: 5 5 5 05/30 Read block 577 len 12288 06/30 Read block 324 len 4096 07/30 Read block 326 len 32768 08/30 Read block 213 len 32768 09/30 Read block 190 len 28672 10/30 Flush 11/30 Write block 469 len 20480 data: 4 4 5 6 6 12/30 Read block 30 len 20480 13/30 Write block 3 len 32768 data: 3 3 3 3 3 3 6 7 14/30 Read block 466 len 16384 15/30 Read block 358 len 4096 16/30 Read block 367 len 16384 17/30 Read block 543 len 4096 18/30 Read block 547 len 24576 19/30 Write block 503 len 36864 data: 3 3 3 3 3 3 3 4 4 20/30 Write block 505 len 16384 data: 4 4 4 4 21/30 Read block 517 len 24576 22/30 Read block 442 len 32768 23/30 Read block 302 len 8192 24/30 Write block 593 len 16384 data: 3 4 4 3 25/30 Write block 115 len 8192 data: 4 4 26/30 Write block 434 len 36864 data: 3 3 3 3 4 5 8 8 7 27/30 Write block 264 len 36864 data: 3 4 4 4 3 3 3 4 6 28/30 Write block 193 len 8192 data: 4 5 29/30 Read block 544 len 8192 30/30 Read block 490 len 16384 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 5 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 20 19 20 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:14:59.474504143Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 72 72 72 T T T 1 020-039 39 39 39 76 76 76 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 37 37 37 72 72 72 T T T 6 120-139 37 37 37 72 72 72 F F F 7 140-159 40 40 40 77 77 77 F F F 8 160-179 35 35 35 70 70 70 F F F 9 180-199 36 36 36 71 71 71 T T T 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 37 37 37 72 72 72 T T T 14 280-299 37 37 37 72 72 72 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 40 40 40 77 77 77 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 39 39 39 76 76 76 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 34 34 34 69 69 69 T T T 22 440-459 39 39 39 76 76 76 T T T 23 460-479 26 26 26 52 52 52 T T T 24 480-499 34 34 34 69 69 69 F F F 25 500-519 25 25 25 50 50 50 T T T 26 520-539 36 36 36 71 71 71 F F F 27 540-559 36 36 36 71 71 71 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 33 33 33 66 66 66 T T T Max gen: 40, Max flush: 77 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 39 39 39 76 76 76 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 33 33 33 66 66 66 F F F 5 100-119 41 41 41 78 78 78 F F F 6 120-139 37 37 37 72 72 72 F F F 7 140-159 40 40 40 77 77 77 F F F 8 160-179 35 35 35 70 70 70 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 F F F 12 240-259 32 32 32 65 65 65 F F F 13 260-279 41 41 41 78 78 78 F F F 14 280-299 37 37 37 72 72 72 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 40 40 40 77 77 77 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 39 39 39 76 76 76 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 41 41 41 78 78 78 F F F 22 440-459 41 41 41 78 78 78 F F F 23 460-479 41 41 41 78 78 78 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 36 36 36 71 71 71 F F F 27 540-559 36 36 36 71 71 71 F F F 28 560-579 26 26 26 52 52 52 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 41, Max flush: 78 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.783569803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.82920971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.830160885Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.830205296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.830220414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.830235041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358} {"msg":"Crucible b42340bd-2545-487d-8fbe-f4ade277c643 has session id: 9db185fc-5a4b-4f30-bfee-0e4d96b088cb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.830916355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Upstairs opts: Upstairs UUID: b42340bd-2545-487d-8fbe-f4ade277c643, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.830956597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Crucible stats registered with UUID: b42340bd-2545-487d-8fbe-f4ade277c643","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.830980663Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:03.83099517Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831422111Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831447357Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358} {"msg":"b42340bd-2545-487d-8fbe-f4ade277c643 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831512075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831529301Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"0","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831556746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"0","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831575713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"1","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831601688Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"1","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831626933Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"2","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831653789Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"2","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831678174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831708558Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831735343Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39002)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.831820438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45454)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.832035108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:65466)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.832073981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.832522198Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"2","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.832697435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"0","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.833142473Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"2","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.833302064Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"0","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.833813559Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.833935156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.834019231Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"1","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.834569139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"1","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[0]R flush_numbers[0..12]: [78, 76, 72, 75, 66, 78, 72, 77, 70, 78, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835101441Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[0]R generation[0..12]: [41, 39, 37, 38, 33, 41, 37, 40, 35, 41, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835125386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835147402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[1]R flush_numbers[0..12]: [78, 76, 72, 75, 66, 78, 72, 77, 70, 78, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835164659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[1]R generation[0..12]: [41, 39, 37, 38, 33, 41, 37, 40, 35, 41, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835181226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835198263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[2]R flush_numbers[0..12]: [78, 76, 72, 75, 66, 78, 72, 77, 70, 78, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.83521467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[2]R generation[0..12]: [41, 39, 37, 38, 33, 41, 37, 40, 35, 41, 32, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835231497Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835254972Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835271129Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835287186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835302883Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835318631Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"":"downstairs","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835334708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835349335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"0","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835365512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"1","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835383779Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"client":"2","":"downstairs","session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835400196Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835415043Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"b42340bd-2545-487d-8fbe-f4ade277c643 is now active with session: 9db185fc-5a4b-4f30-bfee-0e4d96b088cb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.83543066Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835445257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358,"session_id":"9db185fc-5a4b-4f30-bfee-0e4d96b088cb"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:03.835464144Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 20480 data: 7 7 5 4 4 02/30 Read block 275 len 40960 03/30 Write block 225 len 36864 data: 2 2 3 4 4 4 4 5 5 04/30 Read block 580 len 8192 05/30 Read block 316 len 20480 06/30 Read block 359 len 24576 07/30 Read block 351 len 24576 08/30 Read block 309 len 36864 09/30 Read block 388 len 32768 10/30 Read block 416 len 16384 11/30 Read block 35 len 32768 12/30 Read block 43 len 12288 13/30 Read block 576 len 40960 14/30 Read block 281 len 28672 15/30 Write block 539 len 20480 data: 4 4 5 8 7 16/30 Read block 511 len 32768 17/30 Write block 170 len 24576 data: 3 3 4 4 4 4 18/30 Write block 29 len 20480 data: 3 3 2 3 3 19/30 Write block 93 len 12288 data: 6 6 5 20/30 Write block 149 len 8192 data: 6 6 21/30 Read block 571 len 28672 22/30 Write block 232 len 8192 data: 6 6 23/30 Read block 174 len 20480 24/30 Write block 20 len 32768 data: 4 3 3 3 3 3 2 2 25/30 Write block 245 len 32768 data: 4 6 6 6 5 4 5 5 26/30 Write block 120 len 40960 data: 7 4 4 3 5 5 5 4 4 4 27/30 Read block 430 len 8192 28/30 Write block 560 len 24576 data: 4 4 3 3 4 5 29/30 Write block 70 len 32768 data: 6 5 6 6 6 6 5 3 30/30 Read block 363 len 36864 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 30 29 28 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:03.857369672Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 39 39 39 76 76 76 T T T 2 040-059 37 37 37 72 72 72 F F F 3 060-079 38 38 38 75 75 75 T T T 4 080-099 33 33 33 66 66 66 T T T 5 100-119 41 41 41 78 78 78 F F F 6 120-139 37 37 37 72 72 72 T T T 7 140-159 40 40 40 77 77 77 T T T 8 160-179 35 35 35 70 70 70 T T T 9 180-199 41 41 41 78 78 78 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 30 30 30 58 58 58 T T T 12 240-259 32 32 32 65 65 65 T T T 13 260-279 41 41 41 78 78 78 F F F 14 280-299 37 37 37 72 72 72 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 40 40 40 77 77 77 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 39 39 39 76 76 76 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 41 41 41 78 78 78 F F F 22 440-459 41 41 41 78 78 78 F F F 23 460-479 41 41 41 78 78 78 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 36 36 36 71 71 71 T T T 27 540-559 36 36 36 71 71 71 T T T 28 560-579 26 26 26 52 52 52 T T T 29 580-599 41 41 41 78 78 78 F F F Max gen: 41, Max flush: 78 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 41 41 41 78 78 78 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 79 79 79 F F F 8 160-179 42 42 42 79 79 79 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 32 32 32 63 63 63 F F F 11 220-239 42 42 42 79 79 79 F F F 12 240-259 42 42 42 79 79 79 F F F 13 260-279 41 41 41 78 78 78 F F F 14 280-299 37 37 37 72 72 72 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 40 40 40 77 77 77 F F F 18 360-379 35 35 35 70 70 70 F F F 19 380-399 39 39 39 76 76 76 F F F 20 400-419 30 30 30 59 59 59 F F F 21 420-439 41 41 41 78 78 78 F F F 22 440-459 41 41 41 78 78 78 F F F 23 460-479 41 41 41 78 78 78 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 42 42 42 79 79 79 F F F 28 560-579 42 42 42 79 79 79 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 42, Max flush: 79 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.150979971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.198145467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.19926045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.199308581Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.199324098Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.199338496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366} {"msg":"Crucible ecb31f1b-adda-4134-b816-4ac85b0e8c8f has session id: 8a347a98-3911-4a22-a5f1-3309483d69e2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.20007117Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Upstairs opts: Upstairs UUID: ecb31f1b-adda-4134-b816-4ac85b0e8c8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200118222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Crucible stats registered with UUID: ecb31f1b-adda-4134-b816-4ac85b0e8c8f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200135768Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:08.200148656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200739967Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.20077341Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366} {"msg":"ecb31f1b-adda-4134-b816-4ac85b0e8c8f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200861314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200884Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"0","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200930961Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"0","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200964185Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"1","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.200994189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"1","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201022764Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"2","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201050879Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"2","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201079744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201109858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201134594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:46178)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201159979Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59233)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201186154Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34235)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.201225687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.202039496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"2","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.202084958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"0","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.202607511Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"0","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.202714981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"2","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.203190163Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.203280107Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.204329263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"1","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.204880051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"1","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[0]R flush_numbers[0..12]: [78, 79, 72, 79, 79, 78, 79, 79, 79, 78, 63, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205459883Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[0]R generation[0..12]: [41, 42, 37, 42, 42, 41, 42, 42, 42, 41, 32, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205485419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205500786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[1]R flush_numbers[0..12]: [78, 79, 72, 79, 79, 78, 79, 79, 79, 78, 63, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205514433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[1]R generation[0..12]: [41, 42, 37, 42, 42, 41, 42, 42, 42, 41, 32, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205527861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205541858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[2]R flush_numbers[0..12]: [78, 79, 72, 79, 79, 78, 79, 79, 79, 78, 63, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205555316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[2]R generation[0..12]: [41, 42, 37, 42, 42, 41, 42, 42, 42, 41, 32, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205575162Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.20558923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205602217Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205614945Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205627503Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-07T18:15:08.205648399Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"":"downstairs","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205661596Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205673334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"0","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205686621Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"1","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205699669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"client":"2","":"downstairs","session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205712677Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205724665Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"ecb31f1b-adda-4134-b816-4ac85b0e8c8f is now active with session: 8a347a98-3911-4a22-a5f1-3309483d69e2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205737072Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.20574868Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366,"session_id":"8a347a98-3911-4a22-a5f1-3309483d69e2"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:08.205760708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 105 len 24576 data: 4 4 4 6 5 5 02/30 Read block 231 len 28672 03/30 Write block 469 len 24576 data: 5 5 6 7 7 6 04/30 Read block 121 len 8192 05/30 Read block 529 len 28672 06/30 Write block 478 len 32768 data: 4 3 3 2 3 4 4 3 07/30 Write block 68 len 16384 data: 4 5 7 6 08/30 Read block 435 len 36864 09/30 Write block 488 len 20480 data: 6 5 5 5 6 10/30 Write block 200 len 32768 data: 3 3 4 3 2 2 3 4 11/30 Write block 357 len 36864 data: 6 6 4 4 3 3 4 4 4 12/30 Read block 448 len 12288 13/30 Read block 47 len 24576 14/30 Read block 34 len 12288 15/30 Write block 555 len 36864 data: 3 4 4 4 4 5 5 4 4 16/30 Read block 323 len 40960 17/30 Flush 18/30 Write block 295 len 28672 data: 6 5 6 4 2 2 2 19/30 Write block 398 len 20480 data: 2 4 4 4 4 20/30 Write block 342 len 16384 data: 4 4 5 4 21/30 Flush 22/30 Write block 369 len 16384 data: 3 3 4 6 23/30 Flush 24/30 Write block 408 len 20480 data: 4 4 4 5 4 25/30 Write block 206 len 16384 data: 4 5 3 3 26/30 Read block 541 len 28672 27/30 Read block 260 len 40960 28/30 Write block 359 len 24576 data: 5 5 4 4 5 5 29/30 Write block 357 len 36864 data: 7 7 6 6 5 5 6 6 5 30/30 Read block 320 len 20480 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:08.228440632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 44 44 44 80 80 80 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 44 44 44 80 80 80 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 79 79 79 F F F 8 160-179 42 42 42 79 79 79 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 44 44 44 80 80 80 T T T 11 220-239 42 42 42 79 79 79 F F F 12 240-259 42 42 42 79 79 79 F F F 13 260-279 41 41 41 78 78 78 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 44 44 44 81 81 81 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 44 44 44 81 81 81 T T T 18 360-379 44 44 44 82 82 82 T T T 19 380-399 44 44 44 81 81 81 F F F 20 400-419 44 44 44 81 81 81 T T T 21 420-439 41 41 41 78 78 78 F F F 22 440-459 41 41 41 78 78 78 F F F 23 460-479 44 44 44 80 80 80 F F F 24 480-499 44 44 44 80 80 80 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 44 44 44 80 80 80 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 44, Max flush: 82 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 44 44 44 80 80 80 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 44 44 44 80 80 80 F F F 6 120-139 42 42 42 79 79 79 F F F 7 140-159 42 42 42 79 79 79 F F F 8 160-179 42 42 42 79 79 79 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 45 45 45 83 83 83 F F F 11 220-239 42 42 42 79 79 79 F F F 12 240-259 42 42 42 79 79 79 F F F 13 260-279 41 41 41 78 78 78 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 44 44 44 81 81 81 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 45 45 45 83 83 83 F F F 18 360-379 45 45 45 83 83 83 F F F 19 380-399 44 44 44 81 81 81 F F F 20 400-419 45 45 45 83 83 83 F F F 21 420-439 41 41 41 78 78 78 F F F 22 440-459 41 41 41 78 78 78 F F F 23 460-479 44 44 44 80 80 80 F F F 24 480-499 44 44 44 80 80 80 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 44 44 44 80 80 80 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 45, Max flush: 83 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.48957219Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.536482063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.537671793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.537730012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.537748159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.537766075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374} {"msg":"Crucible 0abd08ee-b986-414d-a6b4-fe76678c8f03 has session id: 563019a0-d970-4292-aa03-006211cffc01","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.538307525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Upstairs opts: Upstairs UUID: 0abd08ee-b986-414d-a6b4-fe76678c8f03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.538354807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Crucible stats registered with UUID: 0abd08ee-b986-414d-a6b4-fe76678c8f03","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.538382241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:12.538403487Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.53898272Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539012355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374} {"msg":"0abd08ee-b986-414d-a6b4-fe76678c8f03 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.53909281Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539110996Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"0","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539138451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"0","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539152179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"1","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539166156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"1","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539179524Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539193281Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539206179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539258409Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539272897Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:12.539395654Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"ds_connection connected from Ok(127.0.0.1:60094)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539434397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:12.539463182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"ds_connection connected from Ok(127.0.0.1:47356)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539488427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:12.539517971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.539789751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.540197935Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"1","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.540348338Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"0","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.54076814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"1","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.540867782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"0","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.541346513Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:12.541429788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.541187142Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"ds_connection connected from Ok(127.0.0.1:56954)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.541337875Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.543357061Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.544084976Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[0]R flush_numbers[0..12]: [78, 79, 72, 80, 79, 80, 79, 79, 79, 78, 83, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.544951516Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[0]R generation[0..12]: [41, 42, 37, 44, 42, 44, 42, 42, 42, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.544990959Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545011445Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[1]R flush_numbers[0..12]: [78, 79, 72, 80, 79, 80, 79, 79, 79, 78, 83, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545029292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[1]R generation[0..12]: [41, 42, 37, 44, 42, 44, 42, 42, 42, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545047198Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545065185Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[2]R flush_numbers[0..12]: [78, 79, 72, 80, 79, 80, 79, 79, 79, 78, 83, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545082792Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[2]R generation[0..12]: [41, 42, 37, 44, 42, 44, 42, 42, 42, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545099699Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545116915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545133252Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545149799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545165906Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"":"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545192551Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545221016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545236423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"0","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.54525395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"1","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545271197Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"client":"2","":"downstairs","session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545287604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545302741Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"0abd08ee-b986-414d-a6b4-fe76678c8f03 is now active with session: 563019a0-d970-4292-aa03-006211cffc01","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545319058Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545333875Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374,"session_id":"563019a0-d970-4292-aa03-006211cffc01"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:22.545348842Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 16384 data: 4 4 6 6 02/30 Read block 16 len 12288 03/30 Write block 395 len 28672 data: 3 2 2 3 5 5 5 04/30 Write block 266 len 20480 data: 5 5 4 4 4 05/30 Read block 446 len 40960 06/30 Read block 214 len 36864 07/30 Read block 413 len 28672 08/30 Write block 423 len 8192 data: 6 7 09/30 Read block 70 len 8192 10/30 Read block 187 len 40960 11/30 Write block 359 len 12288 data: 7 7 6 12/30 Read block 498 len 8192 13/30 Write block 351 len 36864 data: 3 3 3 3 4 6 8 8 8 14/30 Read block 547 len 20480 15/30 Write block 484 len 20480 data: 5 4 6 6 7 16/30 Read block 575 len 8192 17/30 Write block 69 len 16384 data: 6 8 7 7 18/30 Read block 318 len 32768 19/30 Flush 20/30 Write block 236 len 36864 data: 5 5 5 5 4 4 3 3 3 21/30 Write block 96 len 24576 data: 4 5 4 4 3 3 22/30 Read block 488 len 32768 23/30 Read block 395 len 16384 24/30 Write block 313 len 32768 data: 6 5 5 6 5 5 4 6 25/30 Read block 324 len 28672 26/30 Write block 436 len 20480 data: 4 4 5 6 9 27/30 Write block 222 len 12288 data: 3 3 3 28/30 Read block 196 len 40960 29/30 Write block 127 len 4096 data: 5 30/30 Read block 51 len 28672 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:22.570325252Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 42 42 42 79 79 79 T T T 5 100-119 44 44 44 80 80 80 T T T 6 120-139 42 42 42 79 79 79 T T T 7 140-159 42 42 42 79 79 79 F F F 8 160-179 42 42 42 79 79 79 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 45 45 45 83 83 83 F F F 11 220-239 42 42 42 79 79 79 T T T 12 240-259 42 42 42 79 79 79 T T T 13 260-279 46 46 46 84 84 84 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 44 44 44 81 81 81 T T T 16 320-339 34 34 34 67 67 67 T T T 17 340-359 46 46 46 84 84 84 F F F 18 360-379 46 46 46 84 84 84 F F F 19 380-399 46 46 46 84 84 84 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 46 46 46 84 84 84 T T T 22 440-459 41 41 41 78 78 78 T T T 23 460-479 44 44 44 80 80 80 F F F 24 480-499 46 46 46 84 84 84 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 44 44 44 80 80 80 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 46, Max flush: 84 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 47 47 47 85 85 85 F F F 5 100-119 47 47 47 85 85 85 F F F 6 120-139 47 47 47 85 85 85 F F F 7 140-159 42 42 42 79 79 79 F F F 8 160-179 42 42 42 79 79 79 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 45 45 45 83 83 83 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 47 47 47 85 85 85 F F F 13 260-279 46 46 46 84 84 84 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 47 47 47 85 85 85 F F F 16 320-339 47 47 47 85 85 85 F F F 17 340-359 46 46 46 84 84 84 F F F 18 360-379 46 46 46 84 84 84 F F F 19 380-399 46 46 46 84 84 84 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 47 47 47 85 85 85 F F F 22 440-459 47 47 47 85 85 85 F F F 23 460-479 44 44 44 80 80 80 F F F 24 480-499 46 46 46 84 84 84 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 44 44 44 80 80 80 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 47, Max flush: 85 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.905592715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:26.948949335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.949447693Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.981161156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.98195458Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.981995722Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.982012509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.982030846Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382} {"msg":"Crucible 1c8c9753-7f32-4033-a960-114e47e3c9bc has session id: 68945250-d050-4c65-b010-e8dd10fbedaa","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.98270865Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Upstairs opts: Upstairs UUID: 1c8c9753-7f32-4033-a960-114e47e3c9bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.982750232Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Crucible stats registered with UUID: 1c8c9753-7f32-4033-a960-114e47e3c9bc","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.982768989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:26.982790935Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983274405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.98330378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382} {"msg":"1c8c9753-7f32-4033-a960-114e47e3c9bc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983337974Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983353691Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"0","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983385255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"0","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983402022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"1","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983419249Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"1","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983435626Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"2","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983466Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"2","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983493005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.98351754Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983548595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"ds_connection connected from Ok(127.0.0.1:46728)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983675461Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"ds_connection connected from Ok(127.0.0.1:59578)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983717323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"ds_connection connected from Ok(127.0.0.1:53127)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.983771713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.984501209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"1","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.984537792Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"2","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.984562347Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"0","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.985214597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"1","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.985254969Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"0","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.985285284Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"2","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.985885142Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.985998112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[0]R flush_numbers[0..12]: [78, 79, 72, 84, 85, 85, 85, 79, 79, 78, 83, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986064129Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[0]R generation[0..12]: [41, 42, 37, 46, 47, 47, 47, 42, 42, 41, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986093094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986109361Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[1]R flush_numbers[0..12]: [78, 79, 72, 84, 85, 85, 85, 79, 79, 78, 83, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986123678Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[1]R generation[0..12]: [41, 42, 37, 46, 47, 47, 47, 42, 42, 41, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986137576Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986155723Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[2]R flush_numbers[0..12]: [78, 79, 72, 84, 85, 85, 85, 79, 79, 78, 83, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.98616957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[2]R generation[0..12]: [41, 42, 37, 46, 47, 47, 47, 42, 42, 41, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986183017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:15:26.986196685Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986213662Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986226349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986238947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986251545Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"":"downstairs","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986264512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.98627605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"0","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986289098Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"1","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986301696Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"client":"2","":"downstairs","session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986314393Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986326471Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"1c8c9753-7f32-4033-a960-114e47e3c9bc is now active with session: 68945250-d050-4c65-b010-e8dd10fbedaa","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986338969Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986352846Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382,"session_id":"68945250-d050-4c65-b010-e8dd10fbedaa"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:26.986364984Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 253 len 12288 data: 5 6 6 03/30 Write block 13 len 28672 data: 7 6 7 6 4 4 5 04/30 Read block 27 len 8192 05/30 Read block 117 len 40960 06/30 Read block 520 len 16384 07/30 Read block 260 len 20480 08/30 Write block 60 len 28672 data: 2 2 2 2 3 3 2 09/30 Write block 507 len 4096 data: 5 10/30 Write block 9 len 36864 data: 7 8 8 7 8 7 8 7 5 11/30 Write block 467 len 4096 data: 3 12/30 Write block 391 len 16384 data: 5 7 7 5 13/30 Write block 179 len 4096 data: 7 14/30 Write block 132 len 36864 data: 2 2 2 3 4 4 4 5 2 15/30 Flush 16/30 Write block 348 len 20480 data: 4 4 4 4 4 17/30 Read block 582 len 16384 18/30 Read block 93 len 16384 19/30 Write block 453 len 32768 data: 6 5 6 5 5 5 4 4 20/30 Read block 309 len 24576 21/30 Read block 464 len 20480 22/30 Write block 93 len 20480 data: 7 7 6 5 6 23/30 Read block 503 len 4096 24/30 Write block 424 len 4096 data: 8 25/30 Write block 282 len 12288 data: 3 3 3 26/30 Flush 27/30 Write block 117 len 20480 data: 4 4 5 8 5 28/30 Read block 583 len 24576 29/30 Read block 7 len 36864 30/30 Read block 401 len 16384 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:27.007063215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 48 48 48 87 87 87 F F F 4 080-099 48 48 48 88 88 88 F F F 5 100-119 47 47 47 85 85 85 T T T 6 120-139 48 48 48 87 87 87 T T T 7 140-159 48 48 48 87 87 87 F F F 8 160-179 48 48 48 87 87 87 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 45 45 45 83 83 83 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 48 48 48 87 87 87 F F F 13 260-279 46 46 46 84 84 84 F F F 14 280-299 48 48 48 88 88 88 F F F 15 300-319 47 47 47 85 85 85 F F F 16 320-339 47 47 47 85 85 85 F F F 17 340-359 48 48 48 88 88 88 F F F 18 360-379 46 46 46 84 84 84 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 48 48 48 88 88 88 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 48 48 48 88 88 88 F F F 24 480-499 46 46 46 84 84 84 F F F 25 500-519 48 48 48 87 87 87 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 44 44 44 80 80 80 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 48, Max flush: 88 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 37 37 37 72 72 72 F F F 3 060-079 48 48 48 87 87 87 F F F 4 080-099 48 48 48 88 88 88 F F F 5 100-119 49 49 49 89 89 89 F F F 6 120-139 49 49 49 89 89 89 F F F 7 140-159 48 48 48 87 87 87 F F F 8 160-179 48 48 48 87 87 87 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 45 45 45 83 83 83 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 48 48 48 87 87 87 F F F 13 260-279 46 46 46 84 84 84 F F F 14 280-299 48 48 48 88 88 88 F F F 15 300-319 47 47 47 85 85 85 F F F 16 320-339 47 47 47 85 85 85 F F F 17 340-359 48 48 48 88 88 88 F F F 18 360-379 46 46 46 84 84 84 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 48 48 48 88 88 88 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 48 48 48 88 88 88 F F F 24 480-499 46 46 46 84 84 84 F F F 25 500-519 48 48 48 87 87 87 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 44 44 44 80 80 80 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 49, Max flush: 89 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.276809659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.323767663Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.32481223Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.324860831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.324876138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.324890355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390} {"msg":"Crucible ac4eec94-16f1-4063-8fde-52b469603b5c has session id: 9011f022-53a1-4ec6-b87a-e30f029d3d31","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.325558082Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Upstairs opts: Upstairs UUID: ac4eec94-16f1-4063-8fde-52b469603b5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.325592415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Crucible stats registered with UUID: ac4eec94-16f1-4063-8fde-52b469603b5c","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.325608143Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:31.32562077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326101311Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326135675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390} {"msg":"ac4eec94-16f1-4063-8fde-52b469603b5c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326193804Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326219779Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"0","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326250224Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"0","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326276489Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326305243Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326332658Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"2","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326358784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"2","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326381369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326403855Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.32643075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"ds_connection connected from Ok(127.0.0.1:61870)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326455356Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"ds_connection connected from Ok(127.0.0.1:39416)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326477802Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:31.326605258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:31.32664687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:31.326694821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.326841214Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.327223044Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"2","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.327261367Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"0","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.327803146Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"0","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.327932262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"2","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.328346046Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:31.328537581Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.328188275Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"ds_connection connected from Ok(127.0.0.1:57890)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.328367902Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.329415298Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.329987582Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[0]R flush_numbers[0..12]: [87, 79, 72, 87, 88, 89, 89, 87, 87, 78, 83, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330616086Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[0]R generation[0..12]: [48, 42, 37, 48, 48, 49, 49, 48, 48, 41, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330639492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330655598Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[1]R flush_numbers[0..12]: [87, 79, 72, 87, 88, 89, 89, 87, 87, 78, 83, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330669536Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[1]R generation[0..12]: [48, 42, 37, 48, 48, 49, 49, 48, 48, 41, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330683004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330696441Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[2]R flush_numbers[0..12]: [87, 79, 72, 87, 88, 89, 89, 87, 87, 78, 83, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330709479Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[2]R generation[0..12]: [48, 42, 37, 48, 48, 49, 49, 48, 48, 41, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330722516Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330735834Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330748341Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330761059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330779286Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-03-07T18:15:41.330798542Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"":"downstairs","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.33081179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330823667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"0","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330836865Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"1","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330849893Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"client":"2","":"downstairs","session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.33086291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330874858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"ac4eec94-16f1-4063-8fde-52b469603b5c is now active with session: 9011f022-53a1-4ec6-b87a-e30f029d3d31","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330887356Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330898944Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390,"session_id":"9011f022-53a1-4ec6-b87a-e30f029d3d31"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:41.330911021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 475 len 16384 data: 6 4 4 5 02/30 Write block 333 len 16384 data: 4 4 3 2 03/30 Write block 58 len 40960 data: 2 2 3 3 3 3 4 4 3 2 04/30 Write block 80 len 32768 data: 4 4 5 4 5 6 5 7 05/30 Read block 514 len 24576 06/30 Write block 338 len 28672 data: 2 4 4 4 5 5 6 07/30 Read block 528 len 36864 08/30 Read block 330 len 40960 09/30 Write block 566 len 12288 data: 4 5 5 10/30 Read block 426 len 4096 11/30 Read block 381 len 12288 12/30 Write block 365 len 20480 data: 6 6 5 4 4 13/30 Read block 52 len 24576 14/30 Flush 15/30 Write block 157 len 40960 data: 5 6 4 4 4 4 3 3 3 3 16/30 Write block 202 len 40960 data: 5 4 3 3 5 6 4 4 4 5 17/30 Write block 504 len 12288 data: 4 5 5 18/30 Read block 250 len 4096 19/30 Read block 136 len 36864 20/30 Write block 20 len 8192 data: 5 4 21/30 Read block 230 len 16384 22/30 Write block 349 len 8192 data: 5 5 23/30 Write block 475 len 16384 data: 7 5 5 6 24/30 Read block 384 len 12288 25/30 Flush 26/30 Write block 431 len 4096 data: 4 27/30 Write block 70 len 20480 data: 9 8 8 7 7 28/30 Read block 152 len 36864 29/30 Read block 242 len 40960 30/30 Read block 200 len 8192 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:41.350984678Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 50 50 50 90 90 90 T T T 4 080-099 50 50 50 90 90 90 F F F 5 100-119 49 49 49 89 89 89 F F F 6 120-139 49 49 49 89 89 89 F F F 7 140-159 50 50 50 91 91 91 F F F 8 160-179 50 50 50 91 91 91 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 50 50 50 91 91 91 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 48 48 48 87 87 87 F F F 13 260-279 46 46 46 84 84 84 F F F 14 280-299 48 48 48 88 88 88 F F F 15 300-319 47 47 47 85 85 85 F F F 16 320-339 50 50 50 90 90 90 F F F 17 340-359 50 50 50 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 48 48 48 88 88 88 T T T 22 440-459 48 48 48 88 88 88 F F F 23 460-479 50 50 50 91 91 91 F F F 24 480-499 46 46 46 84 84 84 F F F 25 500-519 50 50 50 91 91 91 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 50 50 50 90 90 90 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 50, Max flush: 91 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 50 50 50 90 90 90 F F F 3 060-079 51 51 51 92 92 92 F F F 4 080-099 50 50 50 90 90 90 F F F 5 100-119 49 49 49 89 89 89 F F F 6 120-139 49 49 49 89 89 89 F F F 7 140-159 50 50 50 91 91 91 F F F 8 160-179 50 50 50 91 91 91 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 50 50 50 91 91 91 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 48 48 48 87 87 87 F F F 13 260-279 46 46 46 84 84 84 F F F 14 280-299 48 48 48 88 88 88 F F F 15 300-319 47 47 47 85 85 85 F F F 16 320-339 50 50 50 90 90 90 F F F 17 340-359 50 50 50 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 50 50 50 91 91 91 F F F 24 480-499 46 46 46 84 84 84 F F F 25 500-519 50 50 50 91 91 91 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 44 44 44 80 80 80 F F F 28 560-579 50 50 50 90 90 90 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 51, Max flush: 92 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.589992428Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.636431248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.637520007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.637569597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.637585674Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.637600972Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398} {"msg":"Crucible d49747ef-a253-4a1e-8588-b861ed3b5d44 has session id: adcd3289-3367-4f39-ab73-8f95aab1969a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.63825671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Upstairs opts: Upstairs UUID: d49747ef-a253-4a1e-8588-b861ed3b5d44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638290714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Crucible stats registered with UUID: d49747ef-a253-4a1e-8588-b861ed3b5d44","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638307631Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:45.638336036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638820786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638857559Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398} {"msg":"d49747ef-a253-4a1e-8588-b861ed3b5d44 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638914079Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638942774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"0","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638973388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"0","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.638988755Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"1","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.639003502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"1","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.63901728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"2","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.639031617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"2","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.639044575Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.639058382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.63907185Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46246)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.639317954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45038)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.639375363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63493)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.639412337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.640093981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"2","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.640267638Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"0","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.640795421Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"2","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.640935635Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"0","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.641426714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.641577926Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.641659051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"1","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642212199Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"1","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[0]R flush_numbers[0..12]: [87, 91, 90, 92, 90, 89, 89, 91, 91, 78, 91, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.64274393Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[0]R generation[0..12]: [48, 50, 50, 51, 50, 49, 49, 50, 50, 41, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642769586Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642785293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[1]R flush_numbers[0..12]: [87, 91, 90, 92, 90, 89, 89, 91, 91, 78, 91, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.64279987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[1]R generation[0..12]: [48, 50, 50, 51, 50, 49, 49, 50, 50, 41, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642813787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642828735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[2]R flush_numbers[0..12]: [87, 91, 90, 92, 90, 89, 89, 91, 91, 78, 91, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642842192Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[2]R generation[0..12]: [48, 50, 50, 51, 50, 49, 49, 50, 50, 41, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642860059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642874306Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:15:45.642887004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.64290519Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642917868Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642930386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"":"downstairs","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642943243Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642954831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"0","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642967729Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"1","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.642980367Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"client":"2","":"downstairs","session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.643009221Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.643031887Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"d49747ef-a253-4a1e-8588-b861ed3b5d44 is now active with session: adcd3289-3367-4f39-ab73-8f95aab1969a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.643046454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.643058222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398,"session_id":"adcd3289-3367-4f39-ab73-8f95aab1969a"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:45.64307005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 16384 data: 4 5 6 6 02/30 Read block 313 len 8192 03/30 Read block 158 len 28672 04/30 Read block 334 len 40960 05/30 Write block 305 len 16384 data: 3 4 4 4 06/30 Write block 172 len 36864 data: 5 5 5 5 4 6 6 8 8 07/30 Write block 524 len 20480 data: 3 4 4 4 4 08/30 Read block 391 len 28672 09/30 Read block 67 len 32768 10/30 Write block 247 len 36864 data: 7 7 6 5 6 6 6 7 7 11/30 Write block 169 len 36864 data: 2 4 4 6 6 6 6 5 7 12/30 Read block 135 len 36864 13/30 Write block 479 len 24576 data: 4 4 3 4 5 6 14/30 Write block 98 len 12288 data: 5 5 4 15/30 Read block 334 len 28672 16/30 Read block 303 len 8192 17/30 Read block 139 len 24576 18/30 Write block 145 len 4096 data: 6 19/30 Read block 222 len 16384 20/30 Read block 245 len 8192 21/30 Write block 475 len 32768 data: 8 6 6 7 5 5 4 5 22/30 Write block 58 len 12288 data: 3 3 4 23/30 Write block 543 len 8192 data: 8 8 24/30 Write block 320 len 28672 data: 7 5 4 4 4 4 5 25/30 Write block 202 len 20480 data: 6 5 4 4 6 26/30 Read block 201 len 4096 27/30 Read block 374 len 8192 28/30 Write block 277 len 32768 data: 5 4 4 4 3 4 4 4 29/30 Read block 143 len 20480 30/30 Read block 123 len 12288 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:15:45.663555281Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 50 50 50 90 90 90 T T T 3 060-079 51 51 51 92 92 92 T T T 4 080-099 50 50 50 90 90 90 T T T 5 100-119 49 49 49 89 89 89 T T T 6 120-139 49 49 49 89 89 89 F F F 7 140-159 50 50 50 91 91 91 T T T 8 160-179 50 50 50 91 91 91 T T T 9 180-199 41 41 41 78 78 78 T T T 10 200-219 50 50 50 91 91 91 T T T 11 220-239 47 47 47 85 85 85 F F F 12 240-259 48 48 48 87 87 87 T T T 13 260-279 46 46 46 84 84 84 T T T 14 280-299 48 48 48 88 88 88 T T T 15 300-319 47 47 47 85 85 85 T T T 16 320-339 50 50 50 90 90 90 T T T 17 340-359 50 50 50 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 50 50 50 91 91 91 T T T 24 480-499 46 46 46 84 84 84 T T T 25 500-519 50 50 50 91 91 91 T T T 26 520-539 42 42 42 79 79 79 T T T 27 540-559 44 44 44 80 80 80 T T T 28 560-579 50 50 50 90 90 90 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 51, Max flush: 92 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 52 52 52 93 93 93 F F F 3 060-079 52 52 52 93 93 93 F F F 4 080-099 52 52 52 93 93 93 F F F 5 100-119 52 52 52 93 93 93 F F F 6 120-139 49 49 49 89 89 89 F F F 7 140-159 52 52 52 93 93 93 F F F 8 160-179 52 52 52 93 93 93 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 52 52 52 93 93 93 F F F 14 280-299 52 52 52 93 93 93 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 50 50 50 91 91 91 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 48 48 48 87 87 87 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 52 52 52 93 93 93 F F F 26 520-539 52 52 52 93 93 93 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 50 50 50 90 90 90 F F F 29 580-599 41 41 41 78 78 78 F F F Max gen: 52, Max flush: 93 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:15:59.986145879Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.032210278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.033209163Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.033262723Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.03327798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.033292797Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406} {"msg":"Crucible b7de052f-1754-4ded-845d-0ddae188db87 has session id: c26c91cc-5e59-4051-9259-3c5f1a9860b4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.033939938Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Upstairs opts: Upstairs UUID: b7de052f-1754-4ded-845d-0ddae188db87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.03398333Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Crucible stats registered with UUID: b7de052f-1754-4ded-845d-0ddae188db87","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034007315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:00.034028501Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034571601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034601135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406} {"msg":"b7de052f-1754-4ded-845d-0ddae188db87 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034676551Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034695438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"0","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034725552Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"0","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.03473971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034754077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034774283Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"2","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.03478885Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"2","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034859108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034897201Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034922216Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65508)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034958359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35417)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.034984744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:00.035230349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:00.035265142Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:00.035282149Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.035421504Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.035809852Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"2","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.035837927Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"0","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.036581049Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"0","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.036614193Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"2","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.037176159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:00.037263013Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.035991338Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64818)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.036107257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.037432072Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038019953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[0]R flush_numbers[0..12]: [87, 91, 93, 93, 93, 93, 89, 93, 93, 93, 93, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038627791Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[0]R generation[0..12]: [48, 50, 52, 52, 52, 52, 49, 52, 52, 52, 52, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038648267Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038663154Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[1]R flush_numbers[0..12]: [87, 91, 93, 93, 93, 93, 89, 93, 93, 93, 93, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038676672Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[1]R generation[0..12]: [48, 50, 52, 52, 52, 52, 49, 52, 52, 52, 52, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038697378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038720683Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[2]R flush_numbers[0..12]: [87, 91, 93, 93, 93, 93, 89, 93, 93, 93, 93, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038745989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[2]R generation[0..12]: [48, 50, 52, 52, 52, 52, 49, 52, 52, 52, 52, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038774713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038799709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038830103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038855789Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038876585Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038890152Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"":"downstairs","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.03890368Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038924486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"0","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038948921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"1","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038973967Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"client":"2","":"downstairs","session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.038991673Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.039004101Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"b7de052f-1754-4ded-845d-0ddae188db87 is now active with session: c26c91cc-5e59-4051-9259-3c5f1a9860b4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.039023438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.039046253Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406,"session_id":"c26c91cc-5e59-4051-9259-3c5f1a9860b4"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:10.039068499Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 69 len 20480 data: 7 10 9 9 8 02/30 Read block 90 len 16384 03/30 Write block 493 len 20480 data: 5 4 4 4 5 04/30 Flush 05/30 Write block 498 len 12288 data: 4 4 4 06/30 Read block 303 len 36864 07/30 Read block 101 len 4096 08/30 Write block 390 len 36864 data: 5 6 8 8 6 4 3 3 4 09/30 Read block 106 len 28672 10/30 Read block 9 len 24576 11/30 Write block 118 len 28672 data: 5 6 9 6 5 4 6 12/30 Write block 346 len 16384 data: 4 4 5 6 13/30 Read block 243 len 40960 14/30 Flush 15/30 Write block 103 len 32768 data: 3 4 5 5 5 7 6 6 16/30 Read block 338 len 20480 17/30 Read block 223 len 32768 18/30 Flush 19/30 Write block 79 len 16384 data: 3 5 5 6 20/30 Read block 31 len 40960 21/30 Write block 60 len 40960 data: 5 4 4 4 5 5 4 3 5 8 22/30 Read block 382 len 28672 23/30 Write block 40 len 40960 data: 5 3 3 3 4 5 5 4 4 4 24/30 Read block 501 len 24576 25/30 Write block 573 len 32768 data: 5 4 4 4 2 3 5 6 26/30 Read block 227 len 20480 27/30 Write block 167 len 4096 data: 3 28/30 Write block 48 len 20480 data: 5 5 4 4 4 29/30 Read block 183 len 16384 30/30 Read block 439 len 20480 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:10.062368509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 52 52 52 93 93 93 T T T 3 060-079 54 54 54 94 94 94 T T T 4 080-099 52 52 52 93 93 93 T T T 5 100-119 54 54 54 96 96 96 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 52 52 52 93 93 93 F F F 8 160-179 52 52 52 93 93 93 T T T 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 52 52 52 93 93 93 F F F 14 280-299 52 52 52 93 93 93 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 54 54 54 95 95 95 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 52 52 52 93 93 93 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 50 50 50 90 90 90 T T T 29 580-599 41 41 41 78 78 78 T T T Max gen: 54, Max flush: 96 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 55 55 55 97 97 97 F F F 3 060-079 55 55 55 97 97 97 F F F 4 080-099 55 55 55 97 97 97 F F F 5 100-119 54 54 54 96 96 96 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 52 52 52 93 93 93 F F F 8 160-179 55 55 55 97 97 97 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 52 52 52 93 93 93 F F F 14 280-299 52 52 52 93 93 93 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 54 54 54 95 95 95 F F F 18 360-379 50 50 50 90 90 90 F F F 19 380-399 54 54 54 95 95 95 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 52 52 52 93 93 93 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 55 55 55 97 97 97 F F F 29 580-599 55 55 55 97 97 97 F F F Max gen: 55, Max flush: 97 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.330975844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.376425516Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.377372361Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.377415643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.37743021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.377444688Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414} {"msg":"Crucible d7264974-5e10-494f-9622-64a265fc848e has session id: f215e575-902a-401a-829e-dc920fa86557","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.377950804Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Upstairs opts: Upstairs UUID: d7264974-5e10-494f-9622-64a265fc848e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.377989047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Crucible stats registered with UUID: d7264974-5e10-494f-9622-64a265fc848e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378010553Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:14.378024041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378448152Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378473227Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414} {"msg":"d7264974-5e10-494f-9622-64a265fc848e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378543394Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378561421Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"0","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378587856Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"0","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378601654Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"1","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378615601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"1","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378628459Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378642486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378655154Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378668401Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378686608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:14.378811605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:14.378850877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"ds_connection connected from Ok(127.0.0.1:41594)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378879472Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:14.378909627Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"ds_connection connected from Ok(127.0.0.1:47670)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.378934562Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.379005939Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.379629384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"0","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.379670226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"1","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.380305228Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"1","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.380345991Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"0","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.380959227Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:14.380984983Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.379598869Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"ds_connection connected from Ok(127.0.0.1:34708)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.379679484Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.380895859Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.381731965Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[0]R flush_numbers[0..12]: [87, 91, 97, 97, 97, 96, 95, 93, 97, 93, 93, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382413109Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[0]R generation[0..12]: [48, 50, 55, 55, 55, 54, 54, 52, 55, 52, 52, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382444093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.38246014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[1]R flush_numbers[0..12]: [87, 91, 97, 97, 97, 96, 95, 93, 97, 93, 93, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382474857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[1]R generation[0..12]: [48, 50, 55, 55, 55, 54, 54, 52, 55, 52, 52, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382488755Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382502592Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[2]R flush_numbers[0..12]: [87, 91, 97, 97, 97, 96, 95, 93, 97, 93, 93, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.38251597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[2]R generation[0..12]: [48, 50, 55, 55, 55, 54, 54, 52, 55, 52, 52, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382529117Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382542605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382555482Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.38256842Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382587426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382600594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"":"downstairs","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382613701Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-07T18:16:24.382633268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"0","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382646406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"1","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382659133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"client":"2","":"downstairs","session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382672071Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382683928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"d7264974-5e10-494f-9622-64a265fc848e is now active with session: f215e575-902a-401a-829e-dc920fa86557","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382696446Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382707964Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414,"session_id":"f215e575-902a-401a-829e-dc920fa86557"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:24.382719842Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 12288 data: 4 4 4 02/30 Read block 193 len 32768 03/30 Write block 342 len 32768 data: 6 6 7 5 5 5 6 7 04/30 Read block 45 len 40960 05/30 Read block 97 len 40960 06/30 Flush 07/30 Write block 92 len 16384 data: 5 8 8 7 08/30 Read block 109 len 32768 09/30 Write block 168 len 12288 data: 3 3 5 10/30 Read block 191 len 16384 11/30 Write block 49 len 4096 data: 6 12/30 Read block 417 len 32768 13/30 Write block 71 len 36864 data: 10 10 9 8 7 6 4 3 4 14/30 Write block 6 len 40960 data: 4 4 4 8 9 9 8 9 8 9 15/30 Read block 31 len 8192 16/30 Read block 377 len 12288 17/30 Write block 356 len 4096 data: 7 18/30 Write block 262 len 4096 data: 5 19/30 Write block 578 len 4096 data: 4 20/30 Read block 122 len 28672 21/30 Read block 562 len 32768 22/30 Read block 407 len 16384 23/30 Read block 399 len 4096 24/30 Write block 288 len 20480 data: 3 5 7 7 4 25/30 Write block 111 len 40960 data: 3 3 4 4 5 5 5 6 7 10 26/30 Write block 200 len 28672 data: 4 4 7 6 5 5 7 27/30 Write block 383 len 40960 data: 4 3 2 3 3 3 5 6 7 9 28/30 Write block 375 len 20480 data: 6 6 4 4 3 29/30 Write block 429 len 32768 data: 6 5 5 3 3 4 4 5 30/30 Read block 200 len 8192 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 8 Sent Done Sent false 1029 Acked Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 22 24 22 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:24.403713209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 87 87 87 T T T 1 020-039 56 56 56 98 98 98 F F F 2 040-059 55 55 55 97 97 97 T T T 3 060-079 55 55 55 97 97 97 T T T 4 080-099 55 55 55 97 97 97 T T T 5 100-119 54 54 54 96 96 96 T T T 6 120-139 54 54 54 95 95 95 T T T 7 140-159 52 52 52 93 93 93 F F F 8 160-179 55 55 55 97 97 97 T T T 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 T T T 11 220-239 47 47 47 85 85 85 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 52 52 52 93 93 93 T T T 14 280-299 52 52 52 93 93 93 T T T 15 300-319 52 52 52 93 93 93 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 56 56 56 98 98 98 T T T 18 360-379 50 50 50 90 90 90 T T T 19 380-399 54 54 54 95 95 95 T T T 20 400-419 46 46 46 84 84 84 F F F 21 420-439 51 51 51 92 92 92 T T T 22 440-459 48 48 48 88 88 88 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 52 52 52 93 93 93 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 55 55 55 97 97 97 T T T 29 580-599 55 55 55 97 97 97 F F F Max gen: 56, Max flush: 98 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 56 56 56 98 98 98 F F F 2 040-059 57 57 57 99 99 99 F F F 3 060-079 57 57 57 99 99 99 F F F 4 080-099 57 57 57 99 99 99 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 57 57 57 99 99 99 F F F 7 140-159 52 52 52 93 93 93 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 57 57 57 99 99 99 F F F 11 220-239 47 47 47 85 85 85 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 57 57 57 99 99 99 F F F 14 280-299 57 57 57 99 99 99 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 57 57 57 99 99 99 F F F 18 360-379 57 57 57 99 99 99 F F F 19 380-399 57 57 57 99 99 99 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 57 57 57 99 99 99 F F F 22 440-459 48 48 48 88 88 88 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 52 52 52 93 93 93 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 57 57 57 99 99 99 F F F 29 580-599 55 55 55 97 97 97 F F F Max gen: 57, Max flush: 99 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.7221769Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.769394735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.77050592Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.7705605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.770579086Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.770597363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422} {"msg":"Crucible 098c1fc8-e736-4210-a007-60d1364cedcd has session id: bf6608c8-1188-45e9-b98c-1b0c3a3aaa95","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771125925Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Upstairs opts: Upstairs UUID: 098c1fc8-e736-4210-a007-60d1364cedcd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771170827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Crucible stats registered with UUID: 098c1fc8-e736-4210-a007-60d1364cedcd","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771194712Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:28.771217738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771678633Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771707328Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422} {"msg":"098c1fc8-e736-4210-a007-60d1364cedcd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771780134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.77179854Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"0","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771826215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"0","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771839923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"1","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.7718536Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"1","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771866698Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"2","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771880715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"2","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771893733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771906971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.771926457Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"ds_connection connected from Ok(127.0.0.1:32793)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.772206085Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"ds_connection connected from Ok(127.0.0.1:44896)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.772316575Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"ds_connection connected from Ok(127.0.0.1:58615)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.772355468Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.77293608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"1","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.772978382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"0","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.773655477Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"1","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.773684971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"0","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.774273303Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.77439891Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.774495202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"2","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775038021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"2","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[0]R flush_numbers[0..12]: [99, 98, 99, 99, 99, 99, 99, 93, 99, 93, 99, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775572593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[0]R generation[0..12]: [57, 56, 57, 57, 57, 57, 57, 52, 57, 52, 57, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775597088Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775613325Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[1]R flush_numbers[0..12]: [99, 98, 99, 99, 99, 99, 99, 93, 99, 93, 99, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775636201Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[1]R generation[0..12]: [57, 56, 57, 57, 57, 57, 57, 52, 57, 52, 57, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775659886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775684682Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[2]R flush_numbers[0..12]: [99, 98, 99, 99, 99, 99, 99, 93, 99, 93, 99, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775708807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[2]R generation[0..12]: [57, 56, 57, 57, 57, 57, 57, 52, 57, 52, 57, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775728613Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775742931Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775758478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:16:28.775782414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775809039Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775833904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"":"downstairs","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775851001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775862969Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"0","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775876246Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"1","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775898212Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"client":"2","":"downstairs","session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775922898Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775950843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"098c1fc8-e736-4210-a007-60d1364cedcd is now active with session: bf6608c8-1188-45e9-b98c-1b0c3a3aaa95","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.77596577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.775982646Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422,"session_id":"bf6608c8-1188-45e9-b98c-1b0c3a3aaa95"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:28.776004133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 455 len 40960 data: 7 6 6 6 5 5 3 3 3 3 02/30 Read block 369 len 4096 03/30 Read block 266 len 4096 04/30 Read block 344 len 36864 05/30 Read block 252 len 32768 06/30 Write block 47 len 4096 data: 5 07/30 Flush 08/30 Write block 332 len 20480 data: 4 5 5 4 3 09/30 Read block 14 len 36864 10/30 Read block 291 len 24576 11/30 Read block 512 len 36864 12/30 Read block 322 len 28672 13/30 Write block 370 len 8192 data: 4 5 14/30 Read block 527 len 4096 15/30 Read block 456 len 4096 16/30 Write block 349 len 12288 data: 8 6 5 17/30 Write block 25 len 20480 data: 5 3 3 2 4 18/30 Write block 279 len 16384 data: 5 5 4 5 19/30 Write block 23 len 40960 data: 5 5 6 4 4 3 5 4 3 4 20/30 Read block 596 len 16384 21/30 Write block 49 len 4096 data: 7 22/30 Write block 215 len 36864 data: 3 5 5 5 5 4 3 4 4 23/30 Read block 96 len 8192 24/30 Write block 336 len 16384 data: 4 2 3 5 25/30 Write block 322 len 8192 data: 5 5 26/30 Read block 403 len 20480 27/30 Write block 354 len 12288 data: 4 5 8 28/30 Read block 270 len 28672 29/30 Read block 305 len 20480 30/30 Write block 49 len 8192 data: 8 5 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:28.795560505Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 56 56 56 98 98 98 T T T 2 040-059 58 58 58 100 100 100 T T T 3 060-079 57 57 57 99 99 99 F F F 4 080-099 57 57 57 99 99 99 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 57 57 57 99 99 99 F F F 7 140-159 52 52 52 93 93 93 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 57 57 57 99 99 99 T T T 11 220-239 47 47 47 85 85 85 T T T 12 240-259 52 52 52 93 93 93 F F F 13 260-279 57 57 57 99 99 99 T T T 14 280-299 57 57 57 99 99 99 T T T 15 300-319 52 52 52 93 93 93 F F F 16 320-339 52 52 52 93 93 93 T T T 17 340-359 57 57 57 99 99 99 T T T 18 360-379 57 57 57 99 99 99 T T T 19 380-399 57 57 57 99 99 99 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 57 57 57 99 99 99 F F F 22 440-459 58 58 58 100 100 100 F F F 23 460-479 58 58 58 100 100 100 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 52 52 52 93 93 93 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 57 57 57 99 99 99 F F F 29 580-599 55 55 55 97 97 97 F F F Max gen: 58, Max flush: 100 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 59 59 59 101 101 101 F F F 2 040-059 59 59 59 101 101 101 F F F 3 060-079 57 57 57 99 99 99 F F F 4 080-099 57 57 57 99 99 99 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 57 57 57 99 99 99 F F F 7 140-159 52 52 52 93 93 93 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 59 59 59 101 101 101 F F F 11 220-239 59 59 59 101 101 101 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 59 59 59 101 101 101 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 59 59 59 101 101 101 F F F 17 340-359 59 59 59 101 101 101 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 57 57 57 99 99 99 F F F 20 400-419 46 46 46 84 84 84 F F F 21 420-439 57 57 57 99 99 99 F F F 22 440-459 58 58 58 100 100 100 F F F 23 460-479 58 58 58 100 100 100 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 54 54 54 95 95 95 F F F 26 520-539 52 52 52 93 93 93 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 57 57 57 99 99 99 F F F 29 580-599 55 55 55 97 97 97 F F F Max gen: 59, Max flush: 101 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.079621671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.127802259Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.129332386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.129388256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.129407182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.129425049Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430} {"msg":"Crucible db387d9e-ccae-4d1f-93fc-5c52f7dde0ef has session id: a25a0489-9ada-4d6b-8309-1e3df32a5edb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130185808Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Upstairs opts: Upstairs UUID: db387d9e-ccae-4d1f-93fc-5c52f7dde0ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.13022766Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Crucible stats registered with UUID: db387d9e-ccae-4d1f-93fc-5c52f7dde0ef","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130251476Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:43.130272252Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130677067Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130702772Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430} {"msg":"db387d9e-ccae-4d1f-93fc-5c52f7dde0ef active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130751923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.13076914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"0","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130795935Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"0","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130814222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"1","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130828569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"1","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130841427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130855604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130868782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130882289Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.130895427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:43.131033221Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:38411)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.131068645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46828)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.13109462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:43.131119475Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:43.131144131Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.131166147Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.131615713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"0","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.131751018Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"1","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.132194626Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"0","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.132326302Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"1","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.132838477Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:43.132926351Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.132981911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57338)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.133198211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.134266633Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.134835758Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[0]R flush_numbers[0..12]: [99, 101, 101, 99, 99, 99, 99, 93, 99, 93, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135458213Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[0]R generation[0..12]: [57, 59, 59, 57, 57, 57, 57, 52, 57, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135490107Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135508253Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[1]R flush_numbers[0..12]: [99, 101, 101, 99, 99, 99, 99, 93, 99, 93, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.13552541Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[1]R generation[0..12]: [57, 59, 59, 57, 57, 57, 57, 52, 57, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135542797Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135566493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[2]R flush_numbers[0..12]: [99, 101, 101, 99, 99, 99, 99, 93, 99, 93, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135584869Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[2]R generation[0..12]: [57, 59, 59, 57, 57, 57, 57, 52, 57, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135601546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135618553Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.1356342Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135650467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135667704Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135691949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"":"downstairs","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135707946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135722594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"0","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135738851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"1","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135755268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"client":"2","":"downstairs","session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135771145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135785982Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"db387d9e-ccae-4d1f-93fc-5c52f7dde0ef is now active with session: a25a0489-9ada-4d6b-8309-1e3df32a5edb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135801549Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135819336Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430,"session_id":"a25a0489-9ada-4d6b-8309-1e3df32a5edb"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:53.135834533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 394 len 12288 data: 7 5 4 02/30 Read block 551 len 24576 03/30 Write block 460 len 40960 data: 6 4 4 4 4 4 4 4 4 6 04/30 Write block 376 len 20480 data: 7 5 5 4 4 05/30 Read block 435 len 8192 06/30 Write block 561 len 32768 data: 6 5 5 5 6 5 6 6 07/30 Flush 08/30 Flush 09/30 Write block 133 len 4096 data: 3 10/30 Write block 41 len 8192 data: 4 4 11/30 Read block 8 len 36864 12/30 Read block 435 len 40960 13/30 Write block 410 len 32768 data: 5 6 5 4 4 4 4 4 14/30 Write block 494 len 12288 data: 5 5 5 15/30 Flush 16/30 Write block 287 len 8192 data: 3 4 17/30 Read block 443 len 40960 18/30 Write block 339 len 24576 data: 6 5 5 7 7 8 19/30 Write block 519 len 28672 data: 3 2 2 2 2 4 5 20/30 Flush 21/30 Write block 401 len 24576 data: 6 5 5 6 8 7 22/30 Read block 501 len 40960 23/30 Write block 120 len 16384 data: 11 7 6 5 24/30 Read block 300 len 4096 25/30 Read block 233 len 16384 26/30 Write block 61 len 8192 data: 5 5 27/30 Read block 447 len 40960 28/30 Write block 364 len 24576 data: 7 7 7 6 5 5 29/30 Read block 102 len 28672 30/30 Write block 375 len 28672 data: 7 8 6 6 5 5 4 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:53.157291804Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 59 59 59 101 101 101 F F F 2 040-059 60 60 60 104 104 104 F F F 3 060-079 57 57 57 99 99 99 T T T 4 080-099 57 57 57 99 99 99 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 60 60 60 104 104 104 T T T 7 140-159 52 52 52 93 93 93 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 59 59 59 101 101 101 F F F 11 220-239 59 59 59 101 101 101 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 60 60 60 105 105 105 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 60 60 60 105 105 105 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 60 60 60 102 102 102 T T T 19 380-399 60 60 60 102 102 102 T T T 20 400-419 60 60 60 104 104 104 T T T 21 420-439 57 57 57 99 99 99 F F F 22 440-459 58 58 58 100 100 100 F F F 23 460-479 60 60 60 102 102 102 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 60 60 60 105 105 105 F F F 26 520-539 60 60 60 105 105 105 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 60 60 60 102 102 102 F F F 29 580-599 55 55 55 97 97 97 F F F Max gen: 60, Max flush: 105 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 59 59 59 101 101 101 F F F 2 040-059 60 60 60 104 104 104 F F F 3 060-079 61 61 61 106 106 106 F F F 4 080-099 57 57 57 99 99 99 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 61 61 61 106 106 106 F F F 7 140-159 52 52 52 93 93 93 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 59 59 59 101 101 101 F F F 11 220-239 59 59 59 101 101 101 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 60 60 60 105 105 105 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 60 60 60 105 105 105 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 61 61 61 106 106 106 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 57 57 57 99 99 99 F F F 22 440-459 58 58 58 100 100 100 F F F 23 460-479 60 60 60 102 102 102 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 60 60 60 105 105 105 F F F 26 520-539 60 60 60 105 105 105 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 60 60 60 102 102 102 F F F 29 580-599 55 55 55 97 97 97 F F F Max gen: 61, Max flush: 106 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.45513517Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.515284254Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.517587998Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.51773871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.517801268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.517825783Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438} {"msg":"Crucible 0dc2d86a-3645-4550-9ada-9bdc79576d63 has session id: f7ef3949-05cb-4ed6-b31f-9c498916cbcf","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.518842855Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Upstairs opts: Upstairs UUID: 0dc2d86a-3645-4550-9ada-9bdc79576d63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.518886987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Crucible stats registered with UUID: 0dc2d86a-3645-4550-9ada-9bdc79576d63","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.518903694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:57.518930949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519711675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519749008Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438} {"msg":"0dc2d86a-3645-4550-9ada-9bdc79576d63 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.51984698Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519878834Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"0","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519913917Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"0","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519929434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"1","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519944831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"1","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519958569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"2","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519973656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"2","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.519997652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.520012429Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.520026456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:51023)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.520307515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:51262)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.520510127Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:36638)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.520551739Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.521282474Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"1","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.521324476Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"0","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.521448813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"2","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.522153723Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"1","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.522183248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"0","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.522251205Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"2","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.522957574Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523073273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[0]R flush_numbers[0..12]: [99, 101, 104, 106, 99, 99, 106, 93, 99, 93, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523202509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[0]R generation[0..12]: [57, 59, 60, 61, 57, 57, 61, 52, 57, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523232083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523247011Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[1]R flush_numbers[0..12]: [99, 101, 104, 106, 99, 99, 106, 93, 99, 93, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523261078Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[1]R generation[0..12]: [57, 59, 60, 61, 57, 57, 61, 52, 57, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523275206Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523289493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[2]R flush_numbers[0..12]: [99, 101, 104, 106, 99, 99, 106, 93, 99, 93, 101, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.52330345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[2]R generation[0..12]: [57, 59, 60, 61, 57, 57, 61, 52, 57, 52, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523316928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523330515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523343063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.52335598Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523369168Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523381726Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"":"downstairs","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:16:57.523394724Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523459971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"0","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523482017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"1","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523495785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"client":"2","":"downstairs","session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523508892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.5235208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"0dc2d86a-3645-4550-9ada-9bdc79576d63 is now active with session: f7ef3949-05cb-4ed6-b31f-9c498916cbcf","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523533788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523545616Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438,"session_id":"f7ef3949-05cb-4ed6-b31f-9c498916cbcf"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:16:57.523557753Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 367 len 16384 data: 7 6 6 5 03/30 Write block 82 len 36864 data: 7 5 6 7 6 8 6 6 5 04/30 Write block 203 len 8192 data: 7 6 05/30 Write block 480 len 24576 data: 6 5 6 6 7 5 06/30 Write block 577 len 4096 data: 3 07/30 Write block 202 len 24576 data: 8 8 7 6 8 7 08/30 Read block 221 len 24576 09/30 Write block 142 len 24576 data: 2 3 4 7 6 6 10/30 Write block 523 len 36864 data: 3 5 6 5 5 5 3 3 3 11/30 Write block 474 len 12288 data: 7 9 7 12/30 Flush 13/30 Write block 152 len 4096 data: 8 14/30 Read block 35 len 20480 15/30 Write block 449 len 24576 data: 3 4 4 4 7 6 16/30 Write block 30 len 16384 data: 5 4 5 4 17/30 Write block 485 len 32768 data: 6 7 7 8 6 6 6 7 18/30 Write block 72 len 28672 data: 11 10 9 8 7 5 4 19/30 Write block 528 len 8192 data: 6 4 20/30 Read block 95 len 32768 21/30 Read block 94 len 8192 22/30 Read block 48 len 36864 23/30 Read block 368 len 24576 24/30 Read block 596 len 4096 25/30 Read block 413 len 28672 26/30 Write block 594 len 12288 data: 5 5 4 27/30 Write block 559 len 32768 data: 5 6 7 6 6 6 7 6 28/30 Write block 8 len 16384 data: 5 9 10 10 29/30 Read block 211 len 4096 30/30 Write block 278 len 12288 data: 5 6 6 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:16:57.543631Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 T T T 1 020-039 59 59 59 101 101 101 T T T 2 040-059 60 60 60 104 104 104 F F F 3 060-079 61 61 61 106 106 106 T T T 4 080-099 62 62 62 108 108 108 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 61 61 61 106 106 106 F F F 7 140-159 62 62 62 108 108 108 T T T 8 160-179 57 57 57 99 99 99 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 62 62 62 108 108 108 F F F 11 220-239 59 59 59 101 101 101 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 59 59 59 101 101 101 T T T 14 280-299 60 60 60 105 105 105 T T T 15 300-319 52 52 52 93 93 93 F F F 16 320-339 60 60 60 105 105 105 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 57 57 57 99 99 99 F F F 22 440-459 58 58 58 100 100 100 T T T 23 460-479 62 62 62 108 108 108 F F F 24 480-499 62 62 62 108 108 108 T T T 25 500-519 60 60 60 105 105 105 F F F 26 520-539 62 62 62 108 108 108 T T T 27 540-559 52 52 52 93 93 93 T T T 28 560-579 62 62 62 108 108 108 T T T 29 580-599 55 55 55 97 97 97 T T T Max gen: 62, Max flush: 108 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 63 63 63 109 109 109 F F F 2 040-059 60 60 60 104 104 104 F F F 3 060-079 63 63 63 109 109 109 F F F 4 080-099 62 62 62 108 108 108 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 61 61 61 106 106 106 F F F 7 140-159 63 63 63 109 109 109 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 62 62 62 108 108 108 F F F 11 220-239 59 59 59 101 101 101 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 60 60 60 105 105 105 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 57 57 57 99 99 99 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 62 62 62 108 108 108 F F F 24 480-499 63 63 63 109 109 109 F F F 25 500-519 60 60 60 105 105 105 F F F 26 520-539 63 63 63 109 109 109 F F F 27 540-559 63 63 63 109 109 109 F F F 28 560-579 63 63 63 109 109 109 F F F 29 580-599 63 63 63 109 109 109 F F F Max gen: 63, Max flush: 109 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.901147208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.945907198Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.946549279Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.94660117Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.946619606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.946637753Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446} {"msg":"Crucible 29f9c85b-96bb-4230-bfb4-590105d85858 has session id: 01b5c19c-3632-40ba-95c6-a4554be457e5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.94714094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Upstairs opts: Upstairs UUID: 29f9c85b-96bb-4230-bfb4-590105d85858, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947181602Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Crucible stats registered with UUID: 29f9c85b-96bb-4230-bfb4-590105d85858","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947206438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:01.947227954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.94778671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947812255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446} {"msg":"29f9c85b-96bb-4230-bfb4-590105d85858 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947878943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.94789746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"0","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947924605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"0","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947938512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"1","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947953239Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"1","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947972956Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.947999331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.948022287Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.948037154Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.948050231Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48029)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.948162471Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42849)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.948422463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:01.948608088Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:01.9486515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:01.948693383Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.948723557Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.948964992Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"1","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.949003135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"0","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.949676331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"1","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.949709364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"0","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.950277829Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:01.950399507Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.948997167Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37441)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.949103097Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.95033105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.950932469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[0]R flush_numbers[0..12]: [109, 109, 104, 109, 108, 99, 106, 109, 99, 93, 108, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951599035Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[0]R generation[0..12]: [63, 63, 60, 63, 62, 57, 61, 63, 57, 52, 62, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.95162535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951640727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[1]R flush_numbers[0..12]: [109, 109, 104, 109, 108, 99, 106, 109, 99, 93, 108, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951660264Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[1]R generation[0..12]: [63, 63, 60, 63, 62, 57, 61, 63, 57, 52, 62, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951674211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951687999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[2]R flush_numbers[0..12]: [109, 109, 104, 109, 108, 99, 106, 109, 99, 93, 108, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951701326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[2]R generation[0..12]: [63, 63, 60, 63, 62, 57, 61, 63, 57, 52, 62, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951714244Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951727861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951740599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951753426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951765874Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:17:11.951778572Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"":"downstairs","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951796898Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951808456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"0","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951821274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"1","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951833942Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"client":"2","":"downstairs","session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951849419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951861566Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"29f9c85b-96bb-4230-bfb4-590105d85858 is now active with session: 01b5c19c-3632-40ba-95c6-a4554be457e5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951874164Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.951885602Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446,"session_id":"01b5c19c-3632-40ba-95c6-a4554be457e5"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:11.95189738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 21 len 4096 data: 5 02/30 Read block 246 len 36864 03/30 Read block 220 len 40960 04/30 Read block 257 len 32768 05/30 Write block 589 len 8192 data: 5 4 06/30 Write block 69 len 16384 data: 9 11 11 12 07/30 Write block 157 len 40960 data: 6 7 5 5 5 5 4 4 4 4 08/30 Read block 527 len 36864 09/30 Write block 162 len 24576 data: 6 5 5 5 5 4 10/30 Write block 395 len 8192 data: 6 5 11/30 Write block 218 len 8192 data: 6 6 12/30 Write block 166 len 16384 data: 6 5 4 4 13/30 Write block 520 len 24576 data: 3 3 3 4 6 7 14/30 Read block 402 len 28672 15/30 Read block 207 len 12288 16/30 Write block 489 len 40960 data: 7 7 7 8 6 6 6 6 6 5 17/30 Read block 544 len 20480 18/30 Read block 523 len 36864 19/30 Flush 20/30 Flush 21/30 Flush 22/30 Write block 230 len 16384 data: 5 5 7 7 23/30 Write block 433 len 40960 data: 4 5 5 6 5 6 7 10 9 8 24/30 Write block 509 len 24576 data: 4 5 5 5 5 5 25/30 Read block 41 len 36864 26/30 Read block 39 len 36864 27/30 Read block 91 len 36864 28/30 Write block 447 len 36864 data: 5 3 4 5 5 5 8 7 8 29/30 Write block 21 len 24576 data: 6 4 6 6 7 5 30/30 Write block 209 len 20480 data: 5 5 6 4 3 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:11.972577385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 64 64 64 110 110 110 T T T 2 040-059 60 60 60 104 104 104 F F F 3 060-079 64 64 64 110 110 110 F F F 4 080-099 62 62 62 108 108 108 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 61 61 61 106 106 106 F F F 7 140-159 64 64 64 110 110 110 F F F 8 160-179 64 64 64 110 110 110 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 64 64 64 110 110 110 T T T 11 220-239 59 59 59 101 101 101 T T T 12 240-259 52 52 52 93 93 93 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 60 60 60 105 105 105 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 64 64 64 110 110 110 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 57 57 57 99 99 99 T T T 22 440-459 63 63 63 109 109 109 T T T 23 460-479 62 62 62 108 108 108 F F F 24 480-499 64 64 64 110 110 110 F F F 25 500-519 60 60 60 105 105 105 T T T 26 520-539 64 64 64 110 110 110 F F F 27 540-559 63 63 63 109 109 109 F F F 28 560-579 63 63 63 109 109 109 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 64, Max flush: 110 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 65 65 65 111 111 111 F F F 2 040-059 60 60 60 104 104 104 F F F 3 060-079 64 64 64 110 110 110 F F F 4 080-099 62 62 62 108 108 108 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 61 61 61 106 106 106 F F F 7 140-159 64 64 64 110 110 110 F F F 8 160-179 64 64 64 110 110 110 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 65 65 65 111 111 111 F F F 11 220-239 65 65 65 111 111 111 F F F 12 240-259 52 52 52 93 93 93 F F F 13 260-279 63 63 63 109 109 109 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 52 52 52 93 93 93 F F F 16 320-339 60 60 60 105 105 105 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 64 64 64 110 110 110 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 65 65 65 111 111 111 F F F 22 440-459 65 65 65 111 111 111 F F F 23 460-479 62 62 62 108 108 108 F F F 24 480-499 64 64 64 110 110 110 F F F 25 500-519 65 65 65 111 111 111 F F F 26 520-539 64 64 64 110 110 110 F F F 27 540-559 63 63 63 109 109 109 F F F 28 560-579 63 63 63 109 109 109 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 65, Max flush: 111 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.244634801Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.292410233Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.293567669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.293624239Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.293642495Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.293660202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454} {"msg":"Crucible df2c3d2f-3378-42a4-8da5-f35470ff0bae has session id: 864e1820-a6fd-4db2-aeb5-13789c70cab1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.29437393Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Upstairs opts: Upstairs UUID: df2c3d2f-3378-42a4-8da5-f35470ff0bae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.294416342Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Crucible stats registered with UUID: df2c3d2f-3378-42a4-8da5-f35470ff0bae","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.294433369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:26.294446846Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295019831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295049135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454} {"msg":"df2c3d2f-3378-42a4-8da5-f35470ff0bae active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295120422Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295137919Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"0","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295166364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"0","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295180841Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"1","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295202617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"1","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295216954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"2","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295230901Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"2","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295243849Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295257377Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295273434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52691)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295558171Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33048)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295696885Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51253)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.295741507Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.296370311Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"1","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.296402485Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"0","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.297070931Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"1","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.297103085Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"0","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.297735258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.297805445Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.298707838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"2","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299263835Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"2","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[0]R flush_numbers[0..12]: [109, 111, 104, 110, 108, 99, 106, 110, 110, 93, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299814174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[0]R generation[0..12]: [63, 65, 60, 64, 62, 57, 61, 64, 64, 52, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299840689Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299859135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[1]R flush_numbers[0..12]: [109, 111, 104, 110, 108, 99, 106, 110, 110, 93, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299876842Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[1]R generation[0..12]: [63, 65, 60, 64, 62, 57, 61, 64, 64, 52, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299894349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299911996Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[2]R flush_numbers[0..12]: [109, 111, 104, 110, 108, 99, 106, 110, 110, 93, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299929043Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[2]R generation[0..12]: [63, 65, 60, 64, 62, 57, 61, 64, 64, 52, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299945829Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299969325Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.299985432Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300001299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300022625Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300038792Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"":"downstairs","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300054679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:17:26.300069337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"0","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300087323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"1","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.30010387Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"client":"2","":"downstairs","session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300120327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300135464Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"df2c3d2f-3378-42a4-8da5-f35470ff0bae is now active with session: 864e1820-a6fd-4db2-aeb5-13789c70cab1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300151231Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300165639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454,"session_id":"864e1820-a6fd-4db2-aeb5-13789c70cab1"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:26.300180416Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 12288 data: 5 4 3 02/30 Read block 558 len 24576 03/30 Read block 366 len 12288 04/30 Flush 05/30 Write block 194 len 12288 data: 6 4 4 06/30 Read block 46 len 20480 07/30 Read block 238 len 28672 08/30 Write block 261 len 8192 data: 6 6 09/30 Write block 146 len 12288 data: 7 7 7 10/30 Read block 247 len 12288 11/30 Write block 444 len 28672 data: 7 5 7 6 4 5 6 12/30 Write block 255 len 28672 data: 8 5 5 6 6 6 7 13/30 Read block 550 len 32768 14/30 Write block 404 len 12288 data: 7 9 8 15/30 Write block 44 len 12288 data: 5 6 6 16/30 Write block 384 len 40960 data: 4 3 4 4 4 6 7 8 10 9 17/30 Write block 139 len 36864 data: 6 3 2 3 4 5 8 8 8 18/30 Write block 277 len 40960 data: 6 6 7 7 5 6 5 5 3 3 19/30 Write block 179 len 16384 data: 9 9 7 7 20/30 Write block 420 len 20480 data: 6 5 5 7 9 21/30 Read block 355 len 36864 22/30 Read block 55 len 20480 23/30 Flush 24/30 Write block 426 len 4096 data: 6 25/30 Flush 26/30 Write block 49 len 36864 data: 9 6 5 5 4 4 4 4 4 27/30 Write block 318 len 20480 data: 6 5 8 6 6 28/30 Write block 36 len 28672 data: 2 4 4 3 6 5 5 29/30 Write block 479 len 12288 data: 6 7 6 30/30 Read block 255 len 36864 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 5 3 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:26.320102971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 65 65 65 111 111 111 T T T 2 040-059 66 66 66 113 113 113 T T T 3 060-079 64 64 64 110 110 110 F F F 4 080-099 62 62 62 108 108 108 F F F 5 100-119 66 66 66 112 112 112 F F F 6 120-139 66 66 66 113 113 113 F F F 7 140-159 66 66 66 113 113 113 F F F 8 160-179 66 66 66 113 113 113 F F F 9 180-199 66 66 66 113 113 113 F F F 10 200-219 65 65 65 111 111 111 F F F 11 220-239 65 65 65 111 111 111 F F F 12 240-259 66 66 66 113 113 113 F F F 13 260-279 66 66 66 113 113 113 F F F 14 280-299 66 66 66 113 113 113 F F F 15 300-319 52 52 52 93 93 93 T T T 16 320-339 60 60 60 105 105 105 T T T 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 66 66 66 113 113 113 F F F 21 420-439 66 66 66 114 114 114 F F F 22 440-459 66 66 66 113 113 113 F F F 23 460-479 62 62 62 108 108 108 T T T 24 480-499 64 64 64 110 110 110 T T T 25 500-519 65 65 65 111 111 111 F F F 26 520-539 64 64 64 110 110 110 F F F 27 540-559 63 63 63 109 109 109 F F F 28 560-579 63 63 63 109 109 109 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 66, Max flush: 114 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 67 67 67 115 115 115 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 64 64 64 110 110 110 F F F 4 080-099 62 62 62 108 108 108 F F F 5 100-119 66 66 66 112 112 112 F F F 6 120-139 66 66 66 113 113 113 F F F 7 140-159 66 66 66 113 113 113 F F F 8 160-179 66 66 66 113 113 113 F F F 9 180-199 66 66 66 113 113 113 F F F 10 200-219 65 65 65 111 111 111 F F F 11 220-239 65 65 65 111 111 111 F F F 12 240-259 66 66 66 113 113 113 F F F 13 260-279 66 66 66 113 113 113 F F F 14 280-299 66 66 66 113 113 113 F F F 15 300-319 67 67 67 115 115 115 F F F 16 320-339 67 67 67 115 115 115 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 66 66 66 113 113 113 F F F 21 420-439 66 66 66 114 114 114 F F F 22 440-459 66 66 66 113 113 113 F F F 23 460-479 67 67 67 115 115 115 F F F 24 480-499 67 67 67 115 115 115 F F F 25 500-519 65 65 65 111 111 111 F F F 26 520-539 64 64 64 110 110 110 F F F 27 540-559 63 63 63 109 109 109 F F F 28 560-579 63 63 63 109 109 109 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 67, Max flush: 115 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.587974852Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:30.633544892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.63414844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.6656671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.666356082Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.666391256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.666407263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.66642456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462} {"msg":"Crucible ddee4dc4-0e58-4a82-8c5a-c70dcbc2e0c6 has session id: 2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667067151Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Upstairs opts: Upstairs UUID: ddee4dc4-0e58-4a82-8c5a-c70dcbc2e0c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667102734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Crucible stats registered with UUID: ddee4dc4-0e58-4a82-8c5a-c70dcbc2e0c6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.66712587Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:30.667142117Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667711462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667740666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462} {"msg":"ddee4dc4-0e58-4a82-8c5a-c70dcbc2e0c6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667818162Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667840278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"0","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667871652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"0","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667888389Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"1","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667905546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"1","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667921523Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"2","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.66793564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"2","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667948718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667971914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.667997079Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44183)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.668040781Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45353)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.668067026Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54217)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.66809734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.66885877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"2","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.668898892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"1","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.669007612Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"0","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.669565499Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"2","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.669598043Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"1","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.669672579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"0","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670193113Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670297473Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[0]R flush_numbers[0..12]: [109, 115, 115, 110, 108, 112, 113, 113, 113, 113, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.67036771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[0]R generation[0..12]: [63, 67, 67, 64, 62, 66, 66, 66, 66, 66, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670389436Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670404353Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[1]R flush_numbers[0..12]: [109, 115, 115, 110, 108, 112, 113, 113, 113, 113, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670418811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[1]R generation[0..12]: [63, 67, 67, 64, 62, 66, 66, 66, 66, 66, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670432658Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670451285Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[2]R flush_numbers[0..12]: [109, 115, 115, 110, 108, 112, 113, 113, 113, 113, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670465202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[2]R generation[0..12]: [63, 67, 67, 64, 62, 66, 66, 66, 66, 66, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.67047857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670492257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670504595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670525521Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670538349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670550837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"":"downstairs","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670563454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670575092Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"0","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.67058795Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"1","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670600687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"client":"2","":"downstairs","session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670613565Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670625403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"ddee4dc4-0e58-4a82-8c5a-c70dcbc2e0c6 is now active with session: 2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.67063784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670651348Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462,"session_id":"2ecd64df-0b0e-4041-ad6d-5c673aa1b1a7"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:30.670663286Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 220 len 4096 data: 5 02/30 Write block 107 len 20480 data: 6 8 7 7 4 03/30 Read block 193 len 12288 04/30 Write block 72 len 8192 data: 13 11 05/30 Read block 546 len 16384 06/30 Read block 183 len 20480 07/30 Read block 292 len 36864 08/30 Read block 175 len 12288 09/30 Write block 533 len 24576 data: 2 2 2 2 3 4 10/30 Read block 382 len 4096 11/30 Write block 261 len 28672 data: 8 7 5 4 5 6 6 12/30 Flush 13/30 Write block 400 len 4096 data: 6 14/30 Flush 15/30 Write block 27 len 28672 data: 5 4 6 6 5 6 5 16/30 Flush 17/30 Write block 253 len 16384 data: 7 8 9 6 18/30 Flush 19/30 Write block 493 len 32768 data: 7 7 7 7 7 6 5 5 20/30 Read block 205 len 32768 21/30 Read block 9 len 20480 22/30 Write block 495 len 12288 data: 8 8 8 23/30 Read block 305 len 20480 24/30 Read block 349 len 20480 25/30 Flush 26/30 Flush 27/30 Write block 478 len 20480 data: 8 7 8 7 7 28/30 Write block 323 len 24576 data: 6 5 5 6 5 5 29/30 Write block 483 len 24576 data: 7 8 7 8 8 9 30/30 Write block 548 len 16384 data: 5 3 3 2 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Sent Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 2 8 Done 1 1 2 4 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:30.689558061Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 68 68 68 118 118 118 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 68 68 68 116 116 116 F F F 4 080-099 62 62 62 108 108 108 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 66 66 66 113 113 113 F F F 7 140-159 66 66 66 113 113 113 F F F 8 160-179 66 66 66 113 113 113 F F F 9 180-199 66 66 66 113 113 113 F F F 10 200-219 65 65 65 111 111 111 F F F 11 220-239 68 68 68 116 116 116 F F F 12 240-259 68 68 68 119 119 119 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 66 66 66 113 113 113 F F F 15 300-319 67 67 67 115 115 115 F F F 16 320-339 67 67 67 115 115 115 T T T 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 68 68 68 117 117 117 F F F 21 420-439 66 66 66 114 114 114 F F F 22 440-459 66 66 66 113 113 113 F F F 23 460-479 67 67 67 115 115 115 T T T 24 480-499 68 68 68 120 120 120 T T T 25 500-519 68 68 68 120 120 120 F F F 26 520-539 68 68 68 116 116 116 F F F 27 540-559 63 63 63 109 109 109 T T T 28 560-579 63 63 63 109 109 109 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 68, Max flush: 120 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 68 68 68 118 118 118 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 68 68 68 116 116 116 F F F 4 080-099 62 62 62 108 108 108 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 66 66 66 113 113 113 F F F 7 140-159 66 66 66 113 113 113 F F F 8 160-179 66 66 66 113 113 113 F F F 9 180-199 66 66 66 113 113 113 F F F 10 200-219 65 65 65 111 111 111 F F F 11 220-239 68 68 68 116 116 116 F F F 12 240-259 68 68 68 119 119 119 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 66 66 66 113 113 113 F F F 15 300-319 67 67 67 115 115 115 F F F 16 320-339 69 69 69 121 121 121 F F F 17 340-359 60 60 60 105 105 105 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 68 68 68 117 117 117 F F F 21 420-439 66 66 66 114 114 114 F F F 22 440-459 66 66 66 113 113 113 F F F 23 460-479 69 69 69 121 121 121 F F F 24 480-499 69 69 69 121 121 121 F F F 25 500-519 68 68 68 120 120 120 F F F 26 520-539 68 68 68 116 116 116 F F F 27 540-559 69 69 69 121 121 121 F F F 28 560-579 63 63 63 109 109 109 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 69, Max flush: 121 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:34.965072907Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:35.009419373Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.009995137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.042248211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.04327675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.043322932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.043346388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.043364634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470} {"msg":"Crucible 60c1fb2b-6b89-4629-acdc-027f7d5e4f9d has session id: 38b98964-f5ae-48dc-9199-925914dff105","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044023942Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Upstairs opts: Upstairs UUID: 60c1fb2b-6b89-4629-acdc-027f7d5e4f9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044081012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Crucible stats registered with UUID: 60c1fb2b-6b89-4629-acdc-027f7d5e4f9d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044113666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:35.044141041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044544856Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044568382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470} {"msg":"60c1fb2b-6b89-4629-acdc-027f7d5e4f9d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044638189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044655945Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"0","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.04468304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"0","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044696898Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"1","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044712925Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"1","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044735971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"2","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044768325Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"2","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044840741Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044869726Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044895081Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"ds_connection connected from Ok(127.0.0.1:34220)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044917827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"ds_connection connected from Ok(127.0.0.1:40859)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044942112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"ds_connection connected from Ok(127.0.0.1:58219)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.044967638Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.045751133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"1","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.045786096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"2","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.045957785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"0","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.046515861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"1","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.046556444Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"2","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.046617323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"0","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.04711857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047244096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[0]R flush_numbers[0..12]: [109, 118, 115, 116, 108, 116, 113, 113, 113, 113, 111, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047310544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[0]R generation[0..12]: [63, 68, 67, 68, 62, 68, 66, 66, 66, 66, 65, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047329771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047344318Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[1]R flush_numbers[0..12]: [109, 118, 115, 116, 108, 116, 113, 113, 113, 113, 111, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047358455Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[1]R generation[0..12]: [63, 68, 67, 68, 62, 68, 66, 66, 66, 66, 65, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047372473Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.04738641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[2]R flush_numbers[0..12]: [109, 118, 115, 116, 108, 116, 113, 113, 113, 113, 111, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047399958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[2]R generation[0..12]: [63, 68, 67, 68, 62, 68, 66, 66, 66, 66, 65, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047412985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047426703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.04743936Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047452208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:17:35.047464806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047482432Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"":"downstairs","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047499889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047512007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"0","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047525224Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"1","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047538032Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"client":"2","":"downstairs","session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.04755113Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047563257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"60c1fb2b-6b89-4629-acdc-027f7d5e4f9d is now active with session: 38b98964-f5ae-48dc-9199-925914dff105","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047575775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047587483Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470,"session_id":"38b98964-f5ae-48dc-9199-925914dff105"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:35.047599321Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 436 len 32768 data: 7 6 7 8 11 10 9 7 02/30 Read block 386 len 12288 03/30 Flush 04/30 Write block 216 len 12288 data: 6 6 7 05/30 Flush 06/30 Write block 20 len 24576 data: 6 7 5 7 7 8 07/30 Read block 185 len 8192 08/30 Write block 65 len 24576 data: 6 5 4 6 10 12 09/30 Read block 547 len 20480 10/30 Read block 88 len 36864 11/30 Read block 473 len 32768 12/30 Read block 155 len 4096 13/30 Write block 456 len 20480 data: 7 7 7 6 7 14/30 Write block 352 len 20480 data: 5 4 5 6 9 15/30 Write block 18 len 24576 data: 5 6 7 8 6 8 16/30 Read block 310 len 40960 17/30 Write block 83 len 12288 data: 6 7 8 18/30 Write block 88 len 16384 data: 7 7 6 4 19/30 Read block 304 len 8192 20/30 Write block 482 len 16384 data: 8 8 9 8 21/30 Read block 416 len 8192 22/30 Write block 557 len 36864 data: 5 5 6 7 8 7 7 7 8 23/30 Read block 340 len 40960 24/30 Write block 130 len 16384 data: 3 2 3 4 25/30 Read block 392 len 32768 26/30 Read block 334 len 36864 27/30 Write block 301 len 4096 data: 3 28/30 Write block 461 len 8192 data: 5 5 29/30 Write block 291 len 28672 data: 8 5 6 5 7 6 7 30/30 Read block 319 len 20480 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 6 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 24 25 23 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:35.067780178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 T T T 1 020-039 68 68 68 118 118 118 T T T 2 040-059 67 67 67 115 115 115 F F F 3 060-079 68 68 68 116 116 116 T T T 4 080-099 62 62 62 108 108 108 T T T 5 100-119 68 68 68 116 116 116 F F F 6 120-139 66 66 66 113 113 113 T T T 7 140-159 66 66 66 113 113 113 F F F 8 160-179 66 66 66 113 113 113 F F F 9 180-199 66 66 66 113 113 113 F F F 10 200-219 70 70 70 123 123 123 F F F 11 220-239 68 68 68 116 116 116 F F F 12 240-259 68 68 68 119 119 119 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 66 66 66 113 113 113 T T T 15 300-319 67 67 67 115 115 115 T T T 16 320-339 69 69 69 121 121 121 F F F 17 340-359 60 60 60 105 105 105 T T T 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 68 68 68 117 117 117 F F F 21 420-439 70 70 70 122 122 122 F F F 22 440-459 70 70 70 122 122 122 T T T 23 460-479 69 69 69 121 121 121 T T T 24 480-499 69 69 69 121 121 121 T T T 25 500-519 68 68 68 120 120 120 F F F 26 520-539 68 68 68 116 116 116 F F F 27 540-559 69 69 69 121 121 121 T T T 28 560-579 63 63 63 109 109 109 T T T 29 580-599 64 64 64 110 110 110 F F F Max gen: 70, Max flush: 123 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 71 71 71 124 124 124 F F F 4 080-099 71 71 71 124 124 124 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 66 66 66 113 113 113 F F F 8 160-179 66 66 66 113 113 113 F F F 9 180-199 66 66 66 113 113 113 F F F 10 200-219 70 70 70 123 123 123 F F F 11 220-239 68 68 68 116 116 116 F F F 12 240-259 68 68 68 119 119 119 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 71 71 71 124 124 124 F F F 16 320-339 69 69 69 121 121 121 F F F 17 340-359 71 71 71 124 124 124 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 68 68 68 117 117 117 F F F 21 420-439 70 70 70 122 122 122 F F F 22 440-459 71 71 71 124 124 124 F F F 23 460-479 71 71 71 124 124 124 F F F 24 480-499 71 71 71 124 124 124 F F F 25 500-519 68 68 68 120 120 120 F F F 26 520-539 68 68 68 116 116 116 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 71 71 71 124 124 124 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 71, Max flush: 124 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.391625704Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:39.435373092Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.435853673Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.467591552Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.468267627Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.468310899Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.468327166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.468344283Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478} {"msg":"Crucible 08799fdf-6e60-4bdd-8e3c-9ebb15a8d004 has session id: 88736552-da08-4bb6-84aa-e24e767b9930","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.468970327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Upstairs opts: Upstairs UUID: 08799fdf-6e60-4bdd-8e3c-9ebb15a8d004, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469019818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Crucible stats registered with UUID: 08799fdf-6e60-4bdd-8e3c-9ebb15a8d004","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469035755Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:39.4690636Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469503658Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469526934Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478} {"msg":"08799fdf-6e60-4bdd-8e3c-9ebb15a8d004 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469595561Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469612598Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"0","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469639723Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"0","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469653481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"1","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469668128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"1","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469681555Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"2","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469741024Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"2","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469759991Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469795024Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469822289Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"ds_connection connected from Ok(127.0.0.1:63367)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469957514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"ds_connection connected from Ok(127.0.0.1:40133)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.469996417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"ds_connection connected from Ok(127.0.0.1:63540)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.470047977Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.470691269Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"1","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.470720853Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"2","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.470878754Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"0","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.471436351Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"1","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.471465035Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"2","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.471480223Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"0","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472052607Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472077132Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[0]R flush_numbers[0..12]: [124, 124, 115, 124, 124, 116, 124, 113, 113, 113, 123, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472118294Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[0]R generation[0..12]: [71, 71, 67, 71, 71, 68, 71, 66, 66, 66, 70, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472135222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472149599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[1]R flush_numbers[0..12]: [124, 124, 115, 124, 124, 116, 124, 113, 113, 113, 123, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472167296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[1]R generation[0..12]: [71, 71, 67, 71, 71, 68, 71, 66, 66, 66, 70, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472180913Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.47219448Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[2]R flush_numbers[0..12]: [124, 124, 115, 124, 124, 116, 124, 113, 113, 113, 123, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472207998Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[2]R generation[0..12]: [71, 71, 67, 71, 71, 68, 71, 66, 66, 66, 70, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472221435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472235053Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472255079Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472267907Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472280464Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472292692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"":"downstairs","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.47230578Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472317508Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"0","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472330455Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"1","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472343083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"client":"2","":"downstairs","session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472355861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472369938Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"08799fdf-6e60-4bdd-8e3c-9ebb15a8d004 is now active with session: 88736552-da08-4bb6-84aa-e24e767b9930","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472382606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472394094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478,"session_id":"88736552-da08-4bb6-84aa-e24e767b9930"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:39.472405891Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 400 len 16384 data: 7 7 6 6 02/30 Write block 434 len 20480 data: 6 6 8 7 8 03/30 Write block 182 len 4096 data: 8 04/30 Read block 582 len 4096 05/30 Flush 06/30 Write block 178 len 32768 data: 7 10 10 8 9 7 7 7 07/30 Read block 203 len 40960 08/30 Read block 459 len 8192 09/30 Write block 325 len 12288 data: 6 7 6 10/30 Read block 467 len 4096 11/30 Read block 356 len 32768 12/30 Write block 259 len 4096 data: 7 13/30 Write block 537 len 16384 data: 4 5 5 5 14/30 Write block 488 len 36864 data: 10 8 8 8 9 8 8 9 9 15/30 Write block 239 len 8192 data: 6 5 16/30 Read block 456 len 20480 17/30 Write block 552 len 40960 data: 3 3 3 4 5 6 6 7 8 9 18/30 Flush 19/30 Write block 211 len 8192 data: 7 5 20/30 Read block 342 len 40960 21/30 Write block 556 len 8192 data: 6 7 22/30 Read block 443 len 40960 23/30 Read block 353 len 16384 24/30 Write block 505 len 28672 data: 7 7 6 5 5 6 6 25/30 Read block 138 len 4096 26/30 Read block 498 len 20480 27/30 Read block 158 len 40960 28/30 Write block 169 len 40960 data: 5 6 5 7 7 7 7 6 8 8 29/30 Write block 442 len 40960 data: 10 8 8 6 8 7 5 6 7 6 30/30 Write block 502 len 32768 data: 3 5 6 8 8 7 6 6 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:39.49241276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 71 71 71 124 124 124 F F F 4 080-099 71 71 71 124 124 124 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 66 66 66 113 113 113 F F F 8 160-179 72 72 72 126 126 126 T T T 9 180-199 72 72 72 126 126 126 F F F 10 200-219 70 70 70 123 123 123 T T T 11 220-239 72 72 72 126 126 126 F F F 12 240-259 72 72 72 126 126 126 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 71 71 71 124 124 124 F F F 16 320-339 72 72 72 126 126 126 F F F 17 340-359 71 71 71 124 124 124 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 72 72 72 125 125 125 F F F 21 420-439 72 72 72 125 125 125 F F F 22 440-459 71 71 71 124 124 124 T T T 23 460-479 71 71 71 124 124 124 F F F 24 480-499 72 72 72 126 126 126 F F F 25 500-519 68 68 68 120 120 120 T T T 26 520-539 72 72 72 126 126 126 F F F 27 540-559 72 72 72 126 126 126 T T T 28 560-579 72 72 72 126 126 126 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 72, Max flush: 126 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 71 71 71 124 124 124 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 71 71 71 124 124 124 F F F 4 080-099 71 71 71 124 124 124 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 66 66 66 113 113 113 F F F 8 160-179 73 73 73 127 127 127 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 73 73 73 127 127 127 F F F 11 220-239 72 72 72 126 126 126 F F F 12 240-259 72 72 72 126 126 126 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 71 71 71 124 124 124 F F F 16 320-339 72 72 72 126 126 126 F F F 17 340-359 71 71 71 124 124 124 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 72 72 72 125 125 125 F F F 21 420-439 72 72 72 125 125 125 F F F 22 440-459 73 73 73 127 127 127 F F F 23 460-479 71 71 71 124 124 124 F F F 24 480-499 72 72 72 126 126 126 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 73 73 73 127 127 127 F F F 28 560-579 72 72 72 126 126 126 F F F 29 580-599 64 64 64 110 110 110 F F F Max gen: 73, Max flush: 127 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.777067316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.82245799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.82327234Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.82332514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.823340917Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.823355174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486} {"msg":"Crucible f4816869-ac6c-4ded-8868-6cbe74f1c5fc has session id: e2e791f7-9823-4cdd-b618-c05d560013c2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.823907782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Upstairs opts: Upstairs UUID: f4816869-ac6c-4ded-8868-6cbe74f1c5fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.823955013Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Crucible stats registered with UUID: f4816869-ac6c-4ded-8868-6cbe74f1c5fc","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.823983008Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:43.824010203Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824498922Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824539715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486} {"msg":"f4816869-ac6c-4ded-8868-6cbe74f1c5fc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824603923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824630748Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"0","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824659993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"0","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.8246751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"1","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824695346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"1","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824744387Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824820523Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824845798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824861215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.824876123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:43.825027745Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:43.825113019Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:43.825135515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.825192014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:46162)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.825274899Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:39239)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.825295805Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.825902283Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"1","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.826059704Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"0","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.826497273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"1","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.826629328Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"0","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.827097262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:43.827179127Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.826866944Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48363)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.827076585Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.828138909Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.828690957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[0]R flush_numbers[0..12]: [124, 124, 115, 124, 124, 116, 124, 113, 127, 126, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829284467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[0]R generation[0..12]: [71, 71, 67, 71, 71, 68, 71, 66, 73, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829308183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.82932332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[1]R flush_numbers[0..12]: [124, 124, 115, 124, 124, 116, 124, 113, 127, 126, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829337337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[1]R generation[0..12]: [71, 71, 67, 71, 71, 68, 71, 66, 73, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829351995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829366412Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[2]R flush_numbers[0..12]: [124, 124, 115, 124, 124, 116, 124, 113, 127, 126, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829386638Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[2]R generation[0..12]: [71, 71, 67, 71, 71, 68, 71, 66, 73, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829400575Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829414183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:17:53.829426611Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829446077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829458705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829471352Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"":"downstairs","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.82948437Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829496118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"0","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829509025Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"1","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829521993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"client":"2","":"downstairs","session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829534711Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829546739Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"f4816869-ac6c-4ded-8868-6cbe74f1c5fc is now active with session: e2e791f7-9823-4cdd-b618-c05d560013c2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829559136Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829570934Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486,"session_id":"e2e791f7-9823-4cdd-b618-c05d560013c2"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:17:53.829582872Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 142 len 4096 data: 4 02/30 Write block 318 len 8192 data: 7 6 03/30 Write block 440 len 20480 data: 12 11 11 9 9 04/30 Write block 192 len 32768 data: 5 5 7 5 5 4 5 5 05/30 Read block 578 len 8192 06/30 Flush 07/30 Write block 284 len 4096 data: 6 08/30 Write block 572 len 28672 data: 4 6 5 5 5 4 5 09/30 Read block 247 len 4096 10/30 Read block 558 len 20480 11/30 Read block 213 len 32768 12/30 Read block 496 len 8192 13/30 Write block 449 len 36864 data: 7 8 7 6 9 8 9 8 8 14/30 Write block 65 len 40960 data: 7 6 5 7 11 13 12 14 12 10 15/30 Write block 232 len 20480 data: 8 8 4 5 6 16/30 Read block 64 len 36864 17/30 Read block 525 len 20480 18/30 Write block 546 len 32768 data: 6 5 6 4 4 3 4 4 19/30 Write block 190 len 32768 data: 4 4 6 6 8 6 6 5 20/30 Write block 414 len 28672 data: 5 5 5 5 5 6 7 21/30 Write block 331 len 40960 data: 4 5 6 6 5 5 3 4 7 6 22/30 Read block 45 len 32768 23/30 Read block 334 len 12288 24/30 Write block 29 len 24576 data: 7 7 6 7 6 3 25/30 Write block 164 len 36864 data: 6 6 7 6 5 6 7 6 8 26/30 Write block 275 len 4096 data: 3 27/30 Write block 587 len 40960 data: 5 5 6 5 3 3 4 6 6 5 28/30 Write block 437 len 8192 data: 8 9 29/30 Write block 306 len 28672 data: 5 5 5 4 4 5 6 30/30 Read block 287 len 32768 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 24 23 22 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:17:53.847850093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 71 71 71 124 124 124 T T T 2 040-059 67 67 67 115 115 115 F F F 3 060-079 71 71 71 124 124 124 T T T 4 080-099 71 71 71 124 124 124 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 74 74 74 128 128 128 F F F 8 160-179 73 73 73 127 127 127 T T T 9 180-199 74 74 74 128 128 128 T T T 10 200-219 73 73 73 127 127 127 F F F 11 220-239 72 72 72 126 126 126 T T T 12 240-259 72 72 72 126 126 126 F F F 13 260-279 68 68 68 116 116 116 T T T 14 280-299 71 71 71 124 124 124 T T T 15 300-319 74 74 74 128 128 128 T T T 16 320-339 72 72 72 126 126 126 T T T 17 340-359 71 71 71 124 124 124 T T T 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 72 72 72 125 125 125 T T T 21 420-439 72 72 72 125 125 125 T T T 22 440-459 74 74 74 128 128 128 T T T 23 460-479 71 71 71 124 124 124 F F F 24 480-499 72 72 72 126 126 126 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 73 73 73 127 127 127 T T T 28 560-579 72 72 72 126 126 126 T T T 29 580-599 64 64 64 110 110 110 T T T Max gen: 74, Max flush: 128 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 75 75 75 129 129 129 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 75 75 75 129 129 129 F F F 4 080-099 71 71 71 124 124 124 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 74 74 74 128 128 128 F F F 8 160-179 75 75 75 129 129 129 F F F 9 180-199 75 75 75 129 129 129 F F F 10 200-219 73 73 73 127 127 127 F F F 11 220-239 75 75 75 129 129 129 F F F 12 240-259 72 72 72 126 126 126 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 75 75 75 129 129 129 F F F 15 300-319 75 75 75 129 129 129 F F F 16 320-339 75 75 75 129 129 129 F F F 17 340-359 75 75 75 129 129 129 F F F 18 360-379 62 62 62 108 108 108 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 75 75 75 129 129 129 F F F 21 420-439 75 75 75 129 129 129 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 71 71 71 124 124 124 F F F 24 480-499 72 72 72 126 126 126 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 F F F 28 560-579 75 75 75 129 129 129 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 75, Max flush: 129 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.165519602Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.211712057Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.212515649Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.212558491Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.212572858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.212587125Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494} {"msg":"Crucible ffefd9b8-c5c9-4d92-bc64-7eb421de785c has session id: 108ece0b-cbfc-44fe-afd4-bbd307183b91","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213245054Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Upstairs opts: Upstairs UUID: ffefd9b8-c5c9-4d92-bc64-7eb421de785c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213273169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Crucible stats registered with UUID: ffefd9b8-c5c9-4d92-bc64-7eb421de785c","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213286986Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:08.213299164Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213788123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213811329Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494} {"msg":"ffefd9b8-c5c9-4d92-bc64-7eb421de785c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213897023Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213916019Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"0","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213952243Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"0","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.213978508Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"1","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214009652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"1","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214037007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"2","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214064442Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"2","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214079179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214092787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214148177Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:33279)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214172452Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:52250)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214198317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:52504)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214224813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214924083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"0","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.214952288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"1","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.215586081Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"1","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.215724885Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"0","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.216201757Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.216316166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.216656543Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"2","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.21721299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"2","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[0]R flush_numbers[0..12]: [124, 129, 115, 129, 124, 116, 124, 128, 129, 129, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.21775512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[0]R generation[0..12]: [71, 75, 67, 75, 71, 68, 71, 74, 75, 75, 73, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217782624Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217797901Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[1]R flush_numbers[0..12]: [124, 129, 115, 129, 124, 116, 124, 128, 129, 129, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217812329Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[1]R generation[0..12]: [71, 75, 67, 75, 71, 68, 71, 74, 75, 75, 73, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217826186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217840204Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[2]R flush_numbers[0..12]: [124, 129, 115, 129, 124, 116, 124, 128, 129, 129, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217853551Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[2]R generation[0..12]: [71, 75, 67, 75, 71, 68, 71, 74, 75, 75, 73, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217866679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217880476Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217893174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217911651Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":Wait for a query_work_queue command to finish before sending IO 1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217948793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217975229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"":"downstairs","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.217994675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.218006803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"0","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.21801982Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"1","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.218032698Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"client":"2","":"downstairs","session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.218045696Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.218057684Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"ffefd9b8-c5c9-4d92-bc64-7eb421de785c is now active with session: 108ece0b-cbfc-44fe-afd4-bbd307183b91","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.218070511Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.218087418Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494,"session_id":"108ece0b-cbfc-44fe-afd4-bbd307183b91"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:08.218103505Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 367 len 40960 data: 8 7 7 6 6 7 6 6 8 9 02/30 Read block 367 len 36864 03/30 Read block 299 len 20480 04/30 Write block 299 len 16384 data: 3 3 4 2 05/30 Read block 390 len 28672 06/30 Write block 116 len 8192 data: 6 6 07/30 Read block 372 len 4096 08/30 Write block 146 len 24576 data: 9 9 8 7 7 8 09/30 Flush 10/30 Flush 11/30 Write block 203 len 36864 data: 9 8 7 9 8 5 6 6 8 12/30 Write block 251 len 36864 data: 7 7 8 9 10 7 6 7 8 13/30 Write block 284 len 40960 data: 7 4 4 4 5 6 8 9 6 7 14/30 Write block 378 len 20480 data: 7 6 6 5 4 15/30 Write block 212 len 24576 data: 6 4 4 4 7 7 16/30 Read block 40 len 28672 17/30 Read block 306 len 40960 18/30 Write block 469 len 40960 data: 7 6 7 8 8 8 10 8 7 9 19/30 Read block 557 len 24576 20/30 Write block 379 len 36864 data: 7 7 6 5 5 5 4 5 5 21/30 Read block 138 len 20480 22/30 Read block 162 len 8192 23/30 Read block 134 len 20480 24/30 Write block 317 len 12288 data: 6 8 7 25/30 Read block 371 len 28672 26/30 Read block 568 len 24576 27/30 Write block 406 len 36864 data: 9 7 5 5 6 7 6 5 6 28/30 Write block 31 len 40960 data: 7 8 7 4 2 3 5 5 4 7 29/30 Write block 341 len 28672 data: 6 8 8 9 6 6 6 30/30 Read block 145 len 36864 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 9 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 20 20 18 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:08.240325664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 75 75 75 129 129 129 T T T 2 040-059 67 67 67 115 115 115 T T T 3 060-079 75 75 75 129 129 129 F F F 4 080-099 71 71 71 124 124 124 F F F 5 100-119 76 76 76 130 130 130 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 76 76 76 130 130 130 F F F 8 160-179 75 75 75 129 129 129 F F F 9 180-199 75 75 75 129 129 129 F F F 10 200-219 73 73 73 127 127 127 T T T 11 220-239 75 75 75 129 129 129 F F F 12 240-259 72 72 72 126 126 126 T T T 13 260-279 75 75 75 129 129 129 F F F 14 280-299 76 76 76 130 130 130 T T T 15 300-319 76 76 76 130 130 130 T T T 16 320-339 75 75 75 129 129 129 F F F 17 340-359 75 75 75 129 129 129 T T T 18 360-379 76 76 76 130 130 130 T T T 19 380-399 66 66 66 113 113 113 T T T 20 400-419 75 75 75 129 129 129 T T T 21 420-439 75 75 75 129 129 129 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 71 71 71 124 124 124 T T T 24 480-499 72 72 72 126 126 126 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 F F F 28 560-579 75 75 75 129 129 129 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 76, Max flush: 130 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 77 77 77 131 131 131 F F F 2 040-059 77 77 77 131 131 131 F F F 3 060-079 75 75 75 129 129 129 F F F 4 080-099 71 71 71 124 124 124 F F F 5 100-119 76 76 76 130 130 130 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 76 76 76 130 130 130 F F F 8 160-179 75 75 75 129 129 129 F F F 9 180-199 75 75 75 129 129 129 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 75 75 75 129 129 129 F F F 12 240-259 77 77 77 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 77 77 77 131 131 131 F F F 16 320-339 75 75 75 129 129 129 F F F 17 340-359 77 77 77 131 131 131 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 77 77 77 131 131 131 F F F 21 420-439 75 75 75 129 129 129 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 77 77 77 131 131 131 F F F 24 480-499 72 72 72 126 126 126 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 F F F 28 560-579 75 75 75 129 129 129 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 77, Max flush: 131 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.531591207Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:22.5754153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.576022148Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.608130799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.609047499Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.609085212Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.60909892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.609113317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502} {"msg":"Crucible 37e30c9b-c9fa-4aeb-8ad8-bb44aede8d29 has session id: 41680d48-01bf-4405-9734-7077af6cfbeb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.609723084Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Upstairs opts: Upstairs UUID: 37e30c9b-c9fa-4aeb-8ad8-bb44aede8d29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.609792481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Crucible stats registered with UUID: 37e30c9b-c9fa-4aeb-8ad8-bb44aede8d29","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.609820706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:22.609834504Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610271623Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610296008Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502} {"msg":"37e30c9b-c9fa-4aeb-8ad8-bb44aede8d29 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610361496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610379423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"0","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610405248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"0","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610419256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"1","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610433153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"1","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610446031Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"2","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610459808Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"2","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610472646Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610485783Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.61050391Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56158)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610816642Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"ds_connection connected from Ok(127.0.0.1:37446)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610858784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"ds_connection connected from Ok(127.0.0.1:37637)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.610885139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.611345054Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"1","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.611384897Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"2","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.611654647Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"0","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.612147316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"1","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.61217728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"0","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.612192617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"2","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.612758813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.612859334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[0]R flush_numbers[0..12]: [124, 131, 131, 129, 124, 130, 124, 130, 129, 129, 131, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.612978612Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[0]R generation[0..12]: [71, 77, 77, 75, 71, 76, 71, 76, 75, 75, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613000378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613020114Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[1]R flush_numbers[0..12]: [124, 131, 131, 129, 124, 130, 124, 130, 129, 129, 131, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613034292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[1]R generation[0..12]: [71, 77, 77, 75, 71, 76, 71, 76, 75, 75, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613047809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613061397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[2]R flush_numbers[0..12]: [124, 131, 131, 129, 124, 130, 124, 130, 129, 129, 131, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613074664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[2]R generation[0..12]: [71, 77, 77, 75, 71, 76, 71, 76, 75, 75, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613087832Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613101139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Max found gen is 78","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-03-07T18:18:22.613120176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613133314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613145671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613158039Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"":"downstairs","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613170736Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613182304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"0","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613195232Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"1","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613208109Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"client":"2","":"downstairs","session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613223407Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613235415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"37e30c9b-c9fa-4aeb-8ad8-bb44aede8d29 is now active with session: 41680d48-01bf-4405-9734-7077af6cfbeb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613247872Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.61325945Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502,"session_id":"41680d48-01bf-4405-9734-7077af6cfbeb"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:22.613271298Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 36864 data: 9 9 8 7 8 6 6 5 5 02/30 Read block 548 len 8192 03/30 Read block 326 len 8192 04/30 Read block 305 len 8192 05/30 Read block 419 len 8192 06/30 Flush 07/30 Write block 171 len 40960 data: 7 9 8 8 8 7 9 9 11 11 08/30 Write block 78 len 20480 data: 5 5 6 6 8 09/30 Read block 88 len 12288 10/30 Write block 463 len 12288 data: 6 6 5 11/30 Write block 573 len 8192 data: 7 6 12/30 Write block 316 len 24576 data: 7 7 9 8 9 7 13/30 Write block 231 len 12288 data: 6 9 9 14/30 Read block 538 len 12288 15/30 Write block 484 len 16384 data: 10 9 9 9 16/30 Flush 17/30 Write block 113 len 20480 data: 5 5 6 7 7 18/30 Read block 335 len 4096 19/30 Read block 178 len 40960 20/30 Write block 344 len 12288 data: 10 7 7 21/30 Write block 119 len 12288 data: 8 12 8 22/30 Read block 408 len 24576 23/30 Read block 390 len 32768 24/30 Read block 116 len 24576 25/30 Flush 26/30 Write block 226 len 36864 data: 3 4 5 5 6 7 10 10 5 27/30 Read block 441 len 12288 28/30 Read block 386 len 24576 29/30 Write block 140 len 12288 data: 4 3 5 30/30 Read block 71 len 32768 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:22.633726604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 77 77 77 131 131 131 F F F 2 040-059 77 77 77 131 131 131 F F F 3 060-079 78 78 78 133 133 133 F F F 4 080-099 78 78 78 133 133 133 F F F 5 100-119 78 78 78 134 134 134 F F F 6 120-139 78 78 78 134 134 134 F F F 7 140-159 76 76 76 130 130 130 T T T 8 160-179 78 78 78 133 133 133 F F F 9 180-199 78 78 78 133 133 133 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 78 78 78 133 133 133 T T T 12 240-259 77 77 77 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 78 78 78 133 133 133 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 77 77 77 131 131 131 F F F 21 420-439 75 75 75 129 129 129 F F F 22 440-459 78 78 78 132 132 132 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 78 78 78 133 133 133 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 F F F 28 560-579 78 78 78 133 133 133 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 78, Max flush: 134 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 124 124 124 F F F 1 020-039 77 77 77 131 131 131 F F F 2 040-059 77 77 77 131 131 131 F F F 3 060-079 78 78 78 133 133 133 F F F 4 080-099 78 78 78 133 133 133 F F F 5 100-119 78 78 78 134 134 134 F F F 6 120-139 78 78 78 134 134 134 F F F 7 140-159 79 79 79 135 135 135 F F F 8 160-179 78 78 78 133 133 133 F F F 9 180-199 78 78 78 133 133 133 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 79 79 79 135 135 135 F F F 12 240-259 77 77 77 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 78 78 78 133 133 133 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 77 77 77 131 131 131 F F F 21 420-439 75 75 75 129 129 129 F F F 22 440-459 78 78 78 132 132 132 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 78 78 78 133 133 133 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 F F F 28 560-579 78 78 78 133 133 133 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 79, Max flush: 135 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.903143148Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.949724712Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.950558988Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.950603989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.950619366Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.950633744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510} {"msg":"Crucible 2b5d6b17-6b4e-4c13-a1f7-bac721d3a22e has session id: 0cec691d-9658-4d82-8d53-08a30893d696","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951191651Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Upstairs opts: Upstairs UUID: 2b5d6b17-6b4e-4c13-a1f7-bac721d3a22e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951259968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Crucible stats registered with UUID: 2b5d6b17-6b4e-4c13-a1f7-bac721d3a22e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951280714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:26.951293592Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951800958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951834292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510} {"msg":"2b5d6b17-6b4e-4c13-a1f7-bac721d3a22e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.95189476Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951920665Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"0","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.95194876Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"0","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951963238Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"1","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951977845Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"1","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.951997641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.952012359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.952025126Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.952066169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.952096003Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:26.952236047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"ds_connection connected from Ok(127.0.0.1:37993)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.95227747Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:26.952306274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"ds_connection connected from Ok(127.0.0.1:41564)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.952332779Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:26.952359634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.9523837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.952930789Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"1","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.953005705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"0","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.953567851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"1","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.953702376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"0","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.954170309Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:26.954301425Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.953251419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"ds_connection connected from Ok(127.0.0.1:53994)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.953402602Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.95465679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.955316807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[0]R flush_numbers[0..12]: [124, 131, 131, 133, 133, 134, 134, 135, 133, 133, 131, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.955943991Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[0]R generation[0..12]: [71, 77, 77, 78, 78, 78, 78, 79, 78, 78, 77, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.955968207Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.955983454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[1]R flush_numbers[0..12]: [124, 131, 131, 133, 133, 134, 134, 135, 133, 133, 131, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.955997402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[1]R generation[0..12]: [71, 77, 77, 78, 78, 78, 78, 79, 78, 78, 77, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956010779Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956024437Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[2]R flush_numbers[0..12]: [124, 131, 131, 133, 133, 134, 134, 135, 133, 133, 131, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956044273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[2]R generation[0..12]: [71, 77, 77, 78, 78, 78, 78, 79, 78, 78, 77, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.95605814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956071808Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956084435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956097233Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956117059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956130027Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"":"downstairs","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956143304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956155112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"0","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.95616821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"1","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956180968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"client":"2","":"downstairs","session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956193815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956205953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"2b5d6b17-6b4e-4c13-a1f7-bac721d3a22e is now active with session: 0cec691d-9658-4d82-8d53-08a30893d696","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956218621Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956230378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510,"session_id":"0cec691d-9658-4d82-8d53-08a30893d696"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:36.956242236Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 17 len 36864 data: 6 6 7 8 9 7 9 8 9 02/30 Flush 03/30 Flush 04/30 Write block 424 len 32768 data: 10 6 7 6 5 7 6 6 05/30 Flush 06/30 Write block 442 len 4096 data: 12 07/30 Read block 182 len 28672 08/30 Write block 320 len 40960 data: 10 8 7 7 6 7 8 7 6 4 09/30 Read block 318 len 12288 10/30 Read block 198 len 16384 11/30 Read block 351 len 40960 12/30 Read block 346 len 24576 13/30 Write block 34 len 20480 data: 5 3 4 6 6 14/30 Read block 73 len 8192 15/30 Write block 4 len 32768 data: 4 4 5 5 6 10 11 11 16/30 Write block 99 len 20480 data: 6 6 5 4 4 17/30 Read block 98 len 8192 18/30 Read block 446 len 16384 19/30 Read block 165 len 40960 20/30 Read block 133 len 32768 21/30 Write block 228 len 16384 data: 6 6 7 8 22/30 Read block 296 len 36864 23/30 Flush 24/30 Write block 484 len 20480 data: 11 10 10 10 11 25/30 Write block 135 len 24576 data: 4 5 5 5 7 5 26/30 Write block 65 len 8192 data: 8 7 27/30 Read block 24 len 40960 28/30 Read block 510 len 40960 29/30 Read block 146 len 40960 30/30 Write block 17 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:36.977983435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 139 139 139 T T T 1 020-039 80 80 80 139 139 139 F F F 2 040-059 77 77 77 131 131 131 F F F 3 060-079 78 78 78 133 133 133 T T T 4 080-099 80 80 80 139 139 139 F F F 5 100-119 80 80 80 139 139 139 F F F 6 120-139 78 78 78 134 134 134 T T T 7 140-159 79 79 79 135 135 135 T T T 8 160-179 78 78 78 133 133 133 F F F 9 180-199 78 78 78 133 133 133 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 80 80 80 139 139 139 F F F 12 240-259 77 77 77 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 78 78 78 133 133 133 F F F 16 320-339 80 80 80 139 139 139 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 77 77 77 131 131 131 F F F 21 420-439 80 80 80 138 138 138 F F F 22 440-459 80 80 80 139 139 139 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 78 78 78 133 133 133 T T T 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 F F F 28 560-579 78 78 78 133 133 133 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 80, Max flush: 139 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 140 140 140 F F F 1 020-039 80 80 80 139 139 139 F F F 2 040-059 77 77 77 131 131 131 F F F 3 060-079 81 81 81 140 140 140 F F F 4 080-099 80 80 80 139 139 139 F F F 5 100-119 80 80 80 139 139 139 F F F 6 120-139 81 81 81 140 140 140 F F F 7 140-159 81 81 81 140 140 140 F F F 8 160-179 78 78 78 133 133 133 F F F 9 180-199 78 78 78 133 133 133 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 80 80 80 139 139 139 F F F 12 240-259 77 77 77 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 78 78 78 133 133 133 F F F 16 320-339 80 80 80 139 139 139 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 77 77 77 131 131 131 F F F 21 420-439 80 80 80 138 138 138 F F F 22 440-459 80 80 80 139 139 139 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 81 81 81 140 140 140 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 F F F 28 560-579 78 78 78 133 133 133 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 81, Max flush: 140 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.237402349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.283681278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.28475345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.284808849Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.284827436Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.284846522Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518} {"msg":"Crucible 8890b8c6-801f-42fb-9215-09d5cd68d7c9 has session id: d0178f40-75f7-4d91-8fc2-3e2bd112164d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.285520588Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Upstairs opts: Upstairs UUID: 8890b8c6-801f-42fb-9215-09d5cd68d7c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.28556363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Crucible stats registered with UUID: 8890b8c6-801f-42fb-9215-09d5cd68d7c9","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.285582816Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:41.285598963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286129615Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.28615527Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518} {"msg":"8890b8c6-801f-42fb-9215-09d5cd68d7c9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286225497Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286243434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"0","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286271339Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"0","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286296104Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"1","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286322939Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"1","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286348785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286376089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286400915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.28642837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286461384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62413)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286488699Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35195)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286515874Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:41.286545438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:41.286574653Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:41.286602078Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.286717156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.287169313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"0","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.287210775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"1","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.287982872Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"1","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.288122826Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"0","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.288603787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:41.288709268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.288508355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60814)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.288769987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.289842008Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.290473172Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[0]R flush_numbers[0..12]: [140, 139, 131, 140, 139, 139, 140, 140, 133, 133, 131, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291157255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[0]R generation[0..12]: [81, 80, 77, 81, 80, 80, 81, 81, 78, 78, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291190709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291209955Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[1]R flush_numbers[0..12]: [140, 139, 131, 140, 139, 139, 140, 140, 133, 133, 131, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291227012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[1]R generation[0..12]: [81, 80, 77, 81, 80, 80, 81, 81, 78, 78, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291243729Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291261436Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[2]R flush_numbers[0..12]: [140, 139, 131, 140, 139, 139, 140, 140, 133, 133, 131, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291278413Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[2]R generation[0..12]: [81, 80, 77, 81, 80, 80, 81, 81, 78, 78, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:18:51.291312507Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.29134425Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291384123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291416787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291448951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"":"downstairs","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291481415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.29151122Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"0","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291543274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"1","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291572988Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"client":"2","":"downstairs","session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2025-03-07T18:18:51.291597994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291624309Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"8890b8c6-801f-42fb-9215-09d5cd68d7c9 is now active with session: d0178f40-75f7-4d91-8fc2-3e2bd112164d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291640206Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.291654673Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518,"session_id":"d0178f40-75f7-4d91-8fc2-3e2bd112164d"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:51.29166966Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 67 len 32768 data: 6 8 12 14 13 15 13 11 03/30 Flush 04/30 Write block 35 len 4096 data: 4 05/30 Write block 419 len 12288 data: 7 8 6 06/30 Read block 87 len 8192 07/30 Write block 98 len 4096 data: 6 08/30 Read block 9 len 24576 09/30 Read block 61 len 32768 10/30 Read block 434 len 4096 11/30 Flush 12/30 Write block 323 len 32768 data: 8 7 8 9 8 7 5 4 13/30 Write block 574 len 4096 data: 7 14/30 Read block 238 len 24576 15/30 Read block 51 len 28672 16/30 Write block 48 len 28672 data: 6 10 7 6 6 5 5 17/30 Read block 499 len 40960 18/30 Read block 200 len 28672 19/30 Write block 74 len 4096 data: 12 20/30 Read block 129 len 40960 21/30 Write block 12 len 8192 data: 9 10 22/30 Flush 23/30 Write block 98 len 24576 data: 7 7 7 6 5 5 24/30 Read block 487 len 4096 25/30 Write block 550 len 32768 data: 5 4 5 5 4 5 7 8 26/30 Read block 134 len 20480 27/30 Write block 192 len 24576 data: 7 7 9 7 7 6 28/30 Read block 198 len 24576 29/30 Read block 460 len 16384 30/30 Read block 365 len 28672 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:51.312601838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 144 144 144 F F F 1 020-039 82 82 82 143 143 143 F F F 2 040-059 82 82 82 144 144 144 F F F 3 060-079 82 82 82 144 144 144 F F F 4 080-099 82 82 82 143 143 143 T T T 5 100-119 80 80 80 139 139 139 T T T 6 120-139 81 81 81 140 140 140 F F F 7 140-159 81 81 81 140 140 140 F F F 8 160-179 78 78 78 133 133 133 F F F 9 180-199 78 78 78 133 133 133 T T T 10 200-219 77 77 77 131 131 131 F F F 11 220-239 80 80 80 139 139 139 F F F 12 240-259 77 77 77 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 78 78 78 133 133 133 F F F 16 320-339 82 82 82 144 144 144 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 82 82 82 143 143 143 F F F 21 420-439 82 82 82 143 143 143 F F F 22 440-459 80 80 80 139 139 139 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 81 81 81 140 140 140 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 75 75 75 129 129 129 T T T 28 560-579 82 82 82 144 144 144 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 82, Max flush: 144 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 144 144 144 F F F 1 020-039 82 82 82 143 143 143 F F F 2 040-059 82 82 82 144 144 144 F F F 3 060-079 82 82 82 144 144 144 F F F 4 080-099 83 83 83 145 145 145 F F F 5 100-119 83 83 83 145 145 145 F F F 6 120-139 81 81 81 140 140 140 F F F 7 140-159 81 81 81 140 140 140 F F F 8 160-179 78 78 78 133 133 133 F F F 9 180-199 83 83 83 145 145 145 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 80 80 80 139 139 139 F F F 12 240-259 77 77 77 131 131 131 F F F 13 260-279 75 75 75 129 129 129 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 78 78 78 133 133 133 F F F 16 320-339 82 82 82 144 144 144 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 82 82 82 143 143 143 F F F 21 420-439 82 82 82 143 143 143 F F F 22 440-459 80 80 80 139 139 139 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 81 81 81 140 140 140 F F F 25 500-519 73 73 73 127 127 127 F F F 26 520-539 72 72 72 126 126 126 F F F 27 540-559 83 83 83 145 145 145 F F F 28 560-579 82 82 82 144 144 144 F F F 29 580-599 75 75 75 129 129 129 F F F Max gen: 83, Max flush: 145 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.563573005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.609180649Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.609956515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.610010995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.61003788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.610053477Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526} {"msg":"Crucible ac3c0e4f-06f4-4286-9b92-b54ad77b3adb has session id: 69884e65-0f11-40b5-ab67-ae1a4453d70d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.610723073Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Upstairs opts: Upstairs UUID: ac3c0e4f-06f4-4286-9b92-b54ad77b3adb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.610764886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Crucible stats registered with UUID: ac3c0e4f-06f4-4286-9b92-b54ad77b3adb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.610786482Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:55.610799769Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611262963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611293968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526} {"msg":"ac3c0e4f-06f4-4286-9b92-b54ad77b3adb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611345888Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611373063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"0","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611406937Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"0","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611434052Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"1","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611462626Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"1","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611489441Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"2","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611517616Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"2","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611543831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611571566Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611598661Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"ds_connection connected from Ok(127.0.0.1:53932)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611624756Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63051)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611652391Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58835)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.611679456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.612375158Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"0","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.612604685Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"1","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.612959709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"0","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.613105622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"1","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.613127469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"2","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.613537983Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.613646733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.613925211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"2","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[0]R flush_numbers[0..12]: [144, 143, 144, 144, 145, 145, 140, 140, 133, 145, 131, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614477579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[0]R generation[0..12]: [82, 82, 82, 82, 83, 83, 81, 81, 78, 83, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614503164Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614517911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[1]R flush_numbers[0..12]: [144, 143, 144, 144, 145, 145, 140, 140, 133, 145, 131, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614531559Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[1]R generation[0..12]: [82, 82, 82, 82, 83, 83, 81, 81, 78, 83, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614544796Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614558394Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[2]R flush_numbers[0..12]: [144, 143, 144, 144, 145, 145, 140, 140, 133, 145, 131, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614576661Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[2]R generation[0..12]: [82, 82, 82, 82, 83, 83, 81, 81, 78, 83, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614590138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614603645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:18:55.614616213Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.6146317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614644418Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614657026Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"":"downstairs","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614669623Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614681361Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"0","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614694159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"1","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614706757Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"client":"2","":"downstairs","session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614719494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614731482Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"ac3c0e4f-06f4-4286-9b92-b54ad77b3adb is now active with session: 69884e65-0f11-40b5-ab67-ae1a4453d70d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.61474391Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614755527Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526,"session_id":"69884e65-0f11-40b5-ab67-ae1a4453d70d"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:18:55.614767385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 8192 data: 8 9 02/30 Read block 199 len 12288 03/30 Write block 161 len 24576 data: 6 7 6 7 7 8 04/30 Read block 537 len 20480 05/30 Write block 114 len 8192 data: 6 7 06/30 Read block 164 len 24576 07/30 Write block 366 len 12288 data: 8 9 8 08/30 Read block 389 len 40960 09/30 Read block 411 len 20480 10/30 Write block 174 len 4096 data: 9 11/30 Read block 358 len 28672 12/30 Read block 481 len 12288 13/30 Read block 215 len 16384 14/30 Write block 265 len 4096 data: 6 15/30 Write block 412 len 32768 data: 7 6 7 6 6 6 6 8 16/30 Write block 137 len 40960 data: 6 6 8 6 4 6 5 6 9 10 17/30 Flush 18/30 Write block 564 len 8192 data: 8 9 19/30 Read block 10 len 24576 20/30 Read block 233 len 28672 21/30 Write block 439 len 4096 data: 9 22/30 Write block 235 len 40960 data: 6 7 6 6 7 6 5 4 4 4 23/30 Write block 518 len 28672 data: 4 4 4 4 4 5 7 24/30 Write block 539 len 32768 data: 6 6 6 9 9 9 8 7 25/30 Write block 308 len 24576 data: 6 5 5 6 7 7 26/30 Read block 493 len 36864 27/30 Write block 261 len 28672 data: 9 8 6 5 7 7 7 28/30 Write block 574 len 32768 data: 8 6 6 5 6 6 7 7 29/30 Read block 317 len 24576 30/30 Read block 497 len 8192 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:18:55.634084872Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 144 144 144 F F F 1 020-039 82 82 82 143 143 143 F F F 2 040-059 82 82 82 144 144 144 F F F 3 060-079 82 82 82 144 144 144 F F F 4 080-099 83 83 83 145 145 145 F F F 5 100-119 84 84 84 146 146 146 F F F 6 120-139 84 84 84 146 146 146 F F F 7 140-159 84 84 84 146 146 146 F F F 8 160-179 84 84 84 146 146 146 F F F 9 180-199 83 83 83 145 145 145 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 80 80 80 139 139 139 T T T 12 240-259 77 77 77 131 131 131 T T T 13 260-279 84 84 84 146 146 146 T T T 14 280-299 77 77 77 131 131 131 F F F 15 300-319 78 78 78 133 133 133 T T T 16 320-339 84 84 84 146 146 146 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 84 84 84 146 146 146 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 84 84 84 146 146 146 F F F 21 420-439 82 82 82 143 143 143 T T T 22 440-459 80 80 80 139 139 139 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 81 81 81 140 140 140 F F F 25 500-519 73 73 73 127 127 127 T T T 26 520-539 72 72 72 126 126 126 T T T 27 540-559 83 83 83 145 145 145 T T T 28 560-579 82 82 82 144 144 144 T T T 29 580-599 75 75 75 129 129 129 T T T Max gen: 84, Max flush: 146 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 144 144 144 F F F 1 020-039 82 82 82 143 143 143 F F F 2 040-059 82 82 82 144 144 144 F F F 3 060-079 82 82 82 144 144 144 F F F 4 080-099 83 83 83 145 145 145 F F F 5 100-119 84 84 84 146 146 146 F F F 6 120-139 84 84 84 146 146 146 F F F 7 140-159 84 84 84 146 146 146 F F F 8 160-179 84 84 84 146 146 146 F F F 9 180-199 83 83 83 145 145 145 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 85 85 85 147 147 147 F F F 12 240-259 85 85 85 147 147 147 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 84 84 84 146 146 146 F F F 17 340-359 78 78 78 134 134 134 F F F 18 360-379 84 84 84 146 146 146 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 84 84 84 146 146 146 F F F 21 420-439 85 85 85 147 147 147 F F F 22 440-459 80 80 80 139 139 139 F F F 23 460-479 78 78 78 133 133 133 F F F 24 480-499 81 81 81 140 140 140 F F F 25 500-519 85 85 85 147 147 147 F F F 26 520-539 85 85 85 147 147 147 F F F 27 540-559 85 85 85 147 147 147 F F F 28 560-579 85 85 85 147 147 147 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 85, Max flush: 147 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.918755505Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:09.963760429Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.964532047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.996956789Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.998113115Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.998160096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.998184192Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.998203818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534} {"msg":"Crucible 1b3d6347-24fb-4638-b1e3-237a9aaaca38 has session id: 0b141325-6080-46b9-ace0-408509fc9dd1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.998816505Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Upstairs opts: Upstairs UUID: 1b3d6347-24fb-4638-b1e3-237a9aaaca38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.998863136Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Crucible stats registered with UUID: 1b3d6347-24fb-4638-b1e3-237a9aaaca38","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.998890871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:09.998912667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999365423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999392378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534} {"msg":"1b3d6347-24fb-4638-b1e3-237a9aaaca38 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999468784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999486681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"0","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999516315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"0","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999530912Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"1","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.99954562Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"1","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999567166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"2","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999583243Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"2","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999659519Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999691793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.99970736Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43792)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999721697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48457)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999736094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59523)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:09.999807161Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.000672291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"1","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.000859137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"0","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.000891681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"2","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.001371592Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"1","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.001445458Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"2","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.001698931Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"0","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.00202911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002123203Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[0]R flush_numbers[0..12]: [144, 143, 144, 144, 145, 146, 146, 146, 146, 145, 131, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002241211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[0]R generation[0..12]: [82, 82, 82, 82, 83, 84, 84, 84, 84, 83, 77, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002262947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002278154Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[1]R flush_numbers[0..12]: [144, 143, 144, 144, 145, 146, 146, 146, 146, 145, 131, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002291932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[1]R generation[0..12]: [82, 82, 82, 82, 83, 84, 84, 84, 84, 83, 77, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002305299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002318977Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[2]R flush_numbers[0..12]: [144, 143, 144, 144, 145, 146, 146, 146, 146, 145, 131, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002332314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[2]R generation[0..12]: [82, 82, 82, 82, 83, 84, 84, 84, 84, 83, 77, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002345492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002359139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002371757Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:19:10.002384584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002403251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002420708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"":"downstairs","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002434106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002446083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"0","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002459161Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"1","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002472038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"client":"2","":"downstairs","session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002484916Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002496864Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"1b3d6347-24fb-4638-b1e3-237a9aaaca38 is now active with session: 0b141325-6080-46b9-ace0-408509fc9dd1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002509501Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002521239Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534,"session_id":"0b141325-6080-46b9-ace0-408509fc9dd1"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:10.002533137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 108 len 32768 data: 9 8 8 5 4 6 7 8 03/30 Write block 160 len 20480 data: 6 7 8 7 8 04/30 Write block 355 len 32768 data: 7 10 9 9 9 8 7 6 05/30 Read block 324 len 12288 06/30 Flush 07/30 Write block 258 len 28672 data: 8 9 7 10 9 7 6 08/30 Read block 519 len 12288 09/30 Write block 7 len 12288 data: 6 7 11 10/30 Read block 516 len 28672 11/30 Write block 372 len 24576 data: 8 7 7 9 10 7 12/30 Read block 189 len 4096 13/30 Write block 76 len 40960 data: 8 6 6 6 7 7 9 7 8 9 14/30 Read block 120 len 8192 15/30 Write block 84 len 16384 data: 9 10 7 9 16/30 Flush 17/30 Flush 18/30 Write block 498 len 12288 data: 7 6 6 19/30 Read block 227 len 40960 20/30 Write block 66 len 36864 data: 8 7 9 13 15 14 16 14 13 21/30 Read block 243 len 24576 22/30 Write block 431 len 36864 data: 7 4 5 7 7 9 9 10 10 23/30 Write block 238 len 24576 data: 7 8 7 6 5 5 24/30 Write block 401 len 28672 data: 8 7 7 8 10 10 8 25/30 Write block 0 len 32768 data: 2 3 4 4 5 5 6 7 26/30 Flush 27/30 Write block 462 len 32768 data: 7 7 7 6 5 5 5 8 28/30 Write block 122 len 24576 data: 7 6 7 6 6 6 29/30 Read block 373 len 12288 30/30 Write block 490 len 24576 data: 9 9 10 9 9 10 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:10.024266237Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 152 152 152 F F F 1 020-039 82 82 82 143 143 143 F F F 2 040-059 82 82 82 144 144 144 F F F 3 060-079 86 86 86 152 152 152 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 86 86 86 149 149 149 F F F 6 120-139 84 84 84 146 146 146 T T T 7 140-159 84 84 84 146 146 146 F F F 8 160-179 86 86 86 149 149 149 F F F 9 180-199 83 83 83 145 145 145 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 86 86 86 152 152 152 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 84 84 84 146 146 146 F F F 17 340-359 86 86 86 149 149 149 F F F 18 360-379 86 86 86 150 150 150 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 86 86 86 152 152 152 F F F 21 420-439 86 86 86 152 152 152 F F F 22 440-459 80 80 80 139 139 139 F F F 23 460-479 78 78 78 133 133 133 T T T 24 480-499 86 86 86 152 152 152 T T T 25 500-519 86 86 86 152 152 152 F F F 26 520-539 85 85 85 147 147 147 F F F 27 540-559 85 85 85 147 147 147 F F F 28 560-579 85 85 85 147 147 147 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 86, Max flush: 152 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 152 152 152 F F F 1 020-039 82 82 82 143 143 143 F F F 2 040-059 82 82 82 144 144 144 F F F 3 060-079 86 86 86 152 152 152 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 86 86 86 149 149 149 F F F 6 120-139 87 87 87 153 153 153 F F F 7 140-159 84 84 84 146 146 146 F F F 8 160-179 86 86 86 149 149 149 F F F 9 180-199 83 83 83 145 145 145 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 86 86 86 152 152 152 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 84 84 84 146 146 146 F F F 17 340-359 86 86 86 149 149 149 F F F 18 360-379 86 86 86 150 150 150 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 86 86 86 152 152 152 F F F 21 420-439 86 86 86 152 152 152 F F F 22 440-459 80 80 80 139 139 139 F F F 23 460-479 87 87 87 153 153 153 F F F 24 480-499 87 87 87 153 153 153 F F F 25 500-519 86 86 86 152 152 152 F F F 26 520-539 85 85 85 147 147 147 F F F 27 540-559 85 85 85 147 147 147 F F F 28 560-579 85 85 85 147 147 147 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 87, Max flush: 153 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.299201761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.344488144Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.345526952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.345579312Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.345597219Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.345615385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542} {"msg":"Crucible 1d64dd8f-2990-4fef-a51a-9e357cb8db34 has session id: a068e5d0-d437-4d8d-acf2-27e11bdd4755","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346150397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Upstairs opts: Upstairs UUID: 1d64dd8f-2990-4fef-a51a-9e357cb8db34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346206406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Crucible stats registered with UUID: 1d64dd8f-2990-4fef-a51a-9e357cb8db34","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346225523Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:14.34624103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346790408Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346817983Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542} {"msg":"1d64dd8f-2990-4fef-a51a-9e357cb8db34 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346901538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346919764Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"0","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346946089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"0","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346960007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"1","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346973854Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"1","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.346986952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.347000939Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.347016306Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.347039162Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.347063498Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"ds_connection connected from Ok(127.0.0.1:48613)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.347248943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"ds_connection connected from Ok(127.0.0.1:35841)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.347291526Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:14.34748543Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:14.347533861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:14.347566604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.347667646Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.348020851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"0","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.348049635Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"1","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.348708593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"1","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.348749886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"0","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.349435439Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:14.349468973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.349389507Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"ds_connection connected from Ok(127.0.0.1:34753)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.349550128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.350623189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.351243504Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[0]R flush_numbers[0..12]: [152, 143, 144, 152, 150, 149, 153, 146, 149, 145, 131, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.3518661Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[0]R generation[0..12]: [86, 82, 82, 86, 86, 86, 87, 84, 86, 83, 77, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.351893354Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.351913351Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[1]R flush_numbers[0..12]: [152, 143, 144, 152, 150, 149, 153, 146, 149, 145, 131, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.351940026Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[1]R generation[0..12]: [86, 82, 82, 86, 86, 86, 87, 84, 86, 83, 77, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.351958012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.351975579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[2]R flush_numbers[0..12]: [152, 143, 144, 152, 150, 149, 153, 146, 149, 145, 131, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.351992706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[2]R generation[0..12]: [86, 82, 82, 86, 86, 86, 87, 84, 86, 83, 77, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352009193Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352039017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352069812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352118933Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352151657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.35218447Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"":"downstairs","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352216724Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352246999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"0","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:19:24.352279763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"1","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352316016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"client":"2","":"downstairs","session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.35235169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352368616Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"1d64dd8f-2990-4fef-a51a-9e357cb8db34 is now active with session: a068e5d0-d437-4d8d-acf2-27e11bdd4755","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352384813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352399321Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542,"session_id":"a068e5d0-d437-4d8d-acf2-27e11bdd4755"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:24.352414138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 310 len 16384 data: 6 7 8 8 02/30 Write block 201 len 40960 data: 5 9 10 9 8 10 9 6 7 7 03/30 Read block 409 len 16384 04/30 Write block 137 len 4096 data: 7 05/30 Read block 432 len 16384 06/30 Write block 133 len 28672 data: 5 3 5 6 8 7 9 07/30 Write block 502 len 8192 data: 4 6 08/30 Read block 150 len 12288 09/30 Write block 112 len 20480 data: 5 7 8 9 8 10/30 Read block 183 len 24576 11/30 Read block 437 len 16384 12/30 Read block 191 len 40960 13/30 Write block 293 len 20480 data: 8 6 8 7 8 14/30 Write block 446 len 8192 data: 9 8 15/30 Flush 16/30 Write block 25 len 24576 data: 10 6 6 5 8 8 17/30 Read block 9 len 28672 18/30 Write block 428 len 16384 data: 6 8 7 8 19/30 Write block 537 len 24576 data: 5 6 7 7 7 10 20/30 Read block 567 len 24576 21/30 Read block 24 len 28672 22/30 Write block 467 len 8192 data: 6 6 23/30 Read block 352 len 36864 24/30 Read block 171 len 32768 25/30 Write block 286 len 4096 data: 5 26/30 Read block 38 len 12288 27/30 Read block 489 len 24576 28/30 Read block 60 len 28672 29/30 Write block 168 len 16384 data: 6 7 8 8 30/30 Write block 500 len 24576 data: 7 3 5 7 7 9 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:24.373749771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 152 152 152 F F F 1 020-039 82 82 82 143 143 143 T T T 2 040-059 82 82 82 144 144 144 F F F 3 060-079 86 86 86 152 152 152 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 84 84 84 146 146 146 F F F 8 160-179 86 86 86 149 149 149 T T T 9 180-199 83 83 83 145 145 145 F F F 10 200-219 88 88 88 154 154 154 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 86 86 86 152 152 152 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 88 88 88 154 154 154 T T T 15 300-319 88 88 88 154 154 154 F F F 16 320-339 84 84 84 146 146 146 F F F 17 340-359 86 86 86 149 149 149 F F F 18 360-379 86 86 86 150 150 150 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 86 86 86 152 152 152 F F F 21 420-439 86 86 86 152 152 152 T T T 22 440-459 88 88 88 154 154 154 F F F 23 460-479 87 87 87 153 153 153 T T T 24 480-499 87 87 87 153 153 153 F F F 25 500-519 88 88 88 154 154 154 T T T 26 520-539 85 85 85 147 147 147 T T T 27 540-559 85 85 85 147 147 147 T T T 28 560-579 85 85 85 147 147 147 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 88, Max flush: 154 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 152 152 152 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 82 82 82 144 144 144 F F F 3 060-079 86 86 86 152 152 152 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 88 88 88 154 154 154 F F F 7 140-159 84 84 84 146 146 146 F F F 8 160-179 89 89 89 155 155 155 F F F 9 180-199 83 83 83 145 145 145 F F F 10 200-219 88 88 88 154 154 154 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 86 86 86 152 152 152 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 88 88 88 154 154 154 F F F 16 320-339 84 84 84 146 146 146 F F F 17 340-359 86 86 86 149 149 149 F F F 18 360-379 86 86 86 150 150 150 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 86 86 86 152 152 152 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 89 89 89 155 155 155 F F F 24 480-499 87 87 87 153 153 153 F F F 25 500-519 89 89 89 155 155 155 F F F 26 520-539 89 89 89 155 155 155 F F F 27 540-559 89 89 89 155 155 155 F F F 28 560-579 85 85 85 147 147 147 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 89, Max flush: 155 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.655595447Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.701288044Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.702193877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.702240978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.702255655Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.702269883Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550} {"msg":"Crucible 371ab7b7-2ed6-4866-900c-13d0aa5c50e0 has session id: eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.702978102Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Upstairs opts: Upstairs UUID: 371ab7b7-2ed6-4866-900c-13d0aa5c50e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703014005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Crucible stats registered with UUID: 371ab7b7-2ed6-4866-900c-13d0aa5c50e0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703039091Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:38.703061466Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703493776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.70352798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550} {"msg":"371ab7b7-2ed6-4866-900c-13d0aa5c50e0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703563443Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703577561Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"0","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703608645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"0","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703641769Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"1","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703670454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"1","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703729213Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703768196Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703790001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703811028Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703831514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:59502)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.70385256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:55350)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.703876346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:38.704170441Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:38.704218652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:38.704238738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.704325802Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.704729378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"0","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.704760332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"1","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.705492877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"1","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.705629191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"0","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.706065031Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:38.706188378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.705015656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:39880)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.705118947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.706360927Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.70699241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[0]R flush_numbers[0..12]: [152, 155, 144, 152, 150, 154, 154, 146, 155, 145, 154, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707618845Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[0]R generation[0..12]: [86, 89, 82, 86, 86, 88, 88, 84, 89, 83, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.70764495Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707660427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[1]R flush_numbers[0..12]: [152, 155, 144, 152, 150, 154, 154, 146, 155, 145, 154, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707674664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[1]R generation[0..12]: [86, 89, 82, 86, 86, 88, 88, 84, 89, 83, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707688632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707708288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[2]R flush_numbers[0..12]: [152, 155, 144, 152, 150, 154, 154, 146, 155, 145, 154, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707722285Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[2]R generation[0..12]: [86, 89, 82, 86, 86, 88, 88, 84, 89, 83, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707735533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707748961Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707768987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707784594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707800371Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707816118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"":"downstairs","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707832315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707846932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"0","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707863059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"1","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707879426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"client":"2","":"downstairs","session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707895303Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707910191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"371ab7b7-2ed6-4866-900c-13d0aa5c50e0 is now active with session: eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707925928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:48.707943525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:19:48.707958642Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 1 len 20480 data: 4 5 5 6 6 03/30 Read block 367 len 28672 04/30 Flush 05/30 Write block 566 len 12288 data: 7 7 7 06/30 Read block 389 len 20480 07/30 Read block 27 len 40960 08/30 Read block 264 len 20480 09/30 Write block 312 len 12288 data: 9 9 6 10/30 Read block 49 len 40960 11/30 Write block 575 len 32768 data: 7 7 6 7 7 8 8 7 12/30 Read block 189 len 40960 13/30 Read block 503 len 4096 14/30 Read block 124 len 16384 15/30 Read block 355 len 4096 16/30 Write block 139 len 16384 data: 10 7 5 7 17/30 Read block 182 len 40960 18/30 Read block 484 len 40960 19/30 Write block 6 len 24576 data: 7 8 8 12 12 12 20/30 Read block 468 len 36864 21/30 Flush 22/30 Write block 51 len 32768 data: 7 7 6 6 5 5 5 4 23/30 Write block 192 len 28672 data: 8 8 10 8 8 7 6 24/30 Read block 427 len 36864 25/30 Read block 405 len 32768 26/30 Write block 86 len 16384 data: 8 10 8 8 27/30 Write block 353 len 40960 data: 5 6 8 11 10 10 10 9 8 7 28/30 Read block 276 len 8192 29/30 Write block 552 len 12288 data: 6 6 5 30/30 Write block 383 len 32768 data: 6 6 5 6 6 5 7 8 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.729875197Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730474317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73051039Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730531036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730550483Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730575818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730596964Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73061876Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730640276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730662862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730685188Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730708034Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730730689Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730752105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730772391Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730794427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730815164Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73083589Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730849137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730861245Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730873193Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730885221Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730901078Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730913545Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730925543Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730937481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730949489Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730961336Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730973234Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.730985252Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73099728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731009118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731032643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731045771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731057719Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731069547Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731081304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731092932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73110469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731119957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731131895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731143493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731155401Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731167029Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731178776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731190544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731202322Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73121409Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731225758Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731237485Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731249133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731260901Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731272589Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731284217Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731295795Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731307463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.7313213Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731333358Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731345145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731356873Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731368631Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731380279Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731391947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731403705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731415343Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731427041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731438748Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731450396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731462124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731473842Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73148552Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731497317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731509105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731524882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73153696Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731548838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731560496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731572183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731583812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731595529Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731607187Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731618895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731631943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73164379Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731655578Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731667586Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731679434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731691332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731703079Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731715027Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731729094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731741122Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73175296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731764628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731776396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731788074Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731799701Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731811329Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731822907Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731834515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731846123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731857821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731869439Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731880897Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731892514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731904112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.73191559Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731929268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731941055Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731952583Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731964251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731975919Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731987567Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.731999135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732010952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.7320227Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732034448Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732046426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732058334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732070262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732082079Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732093977Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732105915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.732117733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:48.7321313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1550,"session_id":"eeea85e9-8a0a-45b7-b05c-a5b009f9bbbe"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 158 158 158 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 82 82 82 144 144 144 T T T 3 060-079 86 86 86 152 152 152 F F F 4 080-099 86 86 86 150 150 150 T T T 5 100-119 88 88 88 154 154 154 F F F 6 120-139 90 90 90 158 158 158 F F F 7 140-159 90 90 90 158 158 158 F F F 8 160-179 89 89 89 155 155 155 F F F 9 180-199 83 83 83 145 145 145 T T T 10 200-219 88 88 88 154 154 154 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 86 86 86 152 152 152 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 90 90 90 158 158 158 F F F 16 320-339 84 84 84 146 146 146 F F F 17 340-359 86 86 86 149 149 149 T T T 18 360-379 86 86 86 150 150 150 T T T 19 380-399 77 77 77 131 131 131 T T T 20 400-419 86 86 86 152 152 152 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 89 89 89 155 155 155 F F F 24 480-499 87 87 87 153 153 153 F F F 25 500-519 89 89 89 155 155 155 F F F 26 520-539 89 89 89 155 155 155 F F F 27 540-559 89 89 89 155 155 155 T T T 28 560-579 90 90 90 158 158 158 F F F 29 580-599 90 90 90 158 158 158 F F F Max gen: 90, Max flush: 158 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 158 158 158 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 86 86 86 152 152 152 F F F 4 080-099 91 91 91 159 159 159 F F F 5 100-119 88 88 88 154 154 154 F F F 6 120-139 90 90 90 158 158 158 F F F 7 140-159 90 90 90 158 158 158 F F F 8 160-179 89 89 89 155 155 155 F F F 9 180-199 91 91 91 159 159 159 F F F 10 200-219 88 88 88 154 154 154 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 86 86 86 152 152 152 F F F 13 260-279 86 86 86 150 150 150 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 90 90 90 158 158 158 F F F 16 320-339 84 84 84 146 146 146 F F F 17 340-359 91 91 91 159 159 159 F F F 18 360-379 91 91 91 159 159 159 F F F 19 380-399 91 91 91 159 159 159 F F F 20 400-419 86 86 86 152 152 152 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 89 89 89 155 155 155 F F F 24 480-499 87 87 87 153 153 153 F F F 25 500-519 89 89 89 155 155 155 F F F 26 520-539 89 89 89 155 155 155 F F F 27 540-559 91 91 91 159 159 159 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 90 90 90 158 158 158 F F F Max gen: 91, Max flush: 159 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:52.994313723Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.039676481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.04065454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.040701492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.040718179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.040732586Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558} {"msg":"Crucible b02c7e14-25db-4e15-bb1a-f244aacd056d has session id: 05db5e5f-8da6-40d3-abf3-d3ddb0036f3b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.041320597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Upstairs opts: Upstairs UUID: b02c7e14-25db-4e15-bb1a-f244aacd056d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.041356411Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Crucible stats registered with UUID: b02c7e14-25db-4e15-bb1a-f244aacd056d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.041372428Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:53.041391924Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.041891372Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.041927705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558} {"msg":"b02c7e14-25db-4e15-bb1a-f244aacd056d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.041981315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042015319Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"0","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.0420601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"0","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042094654Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"1","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042138916Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"1","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042165391Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042193106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042220061Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042241067Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042265173Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:53.042291858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:53.042318143Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36740)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042342128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:19:53.042368583Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42584)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042391789Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.042418364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.043013424Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"0","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.043057016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"1","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.043689839Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"1","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.043728552Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"0","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.044314634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:19:53.044341019Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.043981675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41919)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.044126618Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.045031801Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.045587018Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[0]R flush_numbers[0..12]: [158, 155, 159, 152, 159, 154, 158, 158, 155, 159, 154, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046256934Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[0]R generation[0..12]: [90, 89, 91, 86, 91, 88, 90, 90, 89, 91, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046284009Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046300386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[1]R flush_numbers[0..12]: [158, 155, 159, 152, 159, 154, 158, 158, 155, 159, 154, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046314744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[1]R generation[0..12]: [90, 89, 91, 86, 91, 88, 90, 90, 89, 91, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046328501Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046349747Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[2]R flush_numbers[0..12]: [158, 155, 159, 152, 159, 154, 158, 158, 155, 159, 154, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046364144Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[2]R generation[0..12]: [90, 89, 91, 86, 91, 88, 90, 90, 89, 91, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046377652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.04639191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046413155Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046437061Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046462406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046480063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"":"downstairs","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046493211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046504818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"0","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046517876Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"1","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046531004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"client":"2","":"downstairs","session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046544021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046555759Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"b02c7e14-25db-4e15-bb1a-f244aacd056d is now active with session: 05db5e5f-8da6-40d3-abf3-d3ddb0036f3b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046568267Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046588133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558,"session_id":"05db5e5f-8da6-40d3-abf3-d3ddb0036f3b"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:03.046600811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 169 len 36864 data: 8 9 9 10 9 10 9 8 10 02/30 Read block 55 len 32768 03/30 Write block 474 len 16384 data: 9 11 9 8 04/30 Write block 213 len 4096 data: 5 05/30 Write block 118 len 16384 data: 7 9 13 9 06/30 Flush 07/30 Write block 326 len 8192 data: 10 9 08/30 Write block 192 len 40960 data: 9 9 11 9 9 8 7 6 5 6 09/30 Write block 486 len 4096 data: 11 10/30 Read block 62 len 40960 11/30 Flush 12/30 Write block 398 len 16384 data: 5 6 8 9 13/30 Read block 75 len 32768 14/30 Write block 353 len 36864 data: 6 7 9 12 11 11 11 10 9 15/30 Read block 134 len 40960 16/30 Write block 257 len 24576 data: 7 9 10 8 11 10 17/30 Read block 250 len 4096 18/30 Write block 332 len 32768 data: 6 7 7 6 6 4 5 8 19/30 Flush 20/30 Write block 116 len 28672 data: 9 8 8 10 14 10 8 21/30 Read block 172 len 12288 22/30 Read block 322 len 24576 23/30 Read block 307 len 24576 24/30 Write block 265 len 16384 data: 8 8 8 5 25/30 Write block 359 len 36864 data: 12 11 10 8 7 8 8 9 10 26/30 Read block 89 len 12288 27/30 Read block 425 len 12288 28/30 Write block 137 len 8192 data: 9 8 29/30 Read block 102 len 36864 30/30 Write block 254 len 32768 data: 10 11 8 8 10 11 9 12 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:03.068559579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 158 158 158 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 86 86 86 152 152 152 F F F 4 080-099 91 91 91 159 159 159 F F F 5 100-119 92 92 92 160 160 160 T T T 6 120-139 92 92 92 160 160 160 T T T 7 140-159 90 90 90 158 158 158 F F F 8 160-179 92 92 92 160 160 160 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 92 92 92 162 162 162 T T T 13 260-279 92 92 92 162 162 162 T T T 14 280-299 89 89 89 155 155 155 F F F 15 300-319 90 90 90 158 158 158 F F F 16 320-339 92 92 92 162 162 162 F F F 17 340-359 92 92 92 162 162 162 T T T 18 360-379 92 92 92 162 162 162 T T T 19 380-399 92 92 92 162 162 162 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 92 92 92 161 161 161 F F F 25 500-519 89 89 89 155 155 155 F F F 26 520-539 89 89 89 155 155 155 F F F 27 540-559 91 91 91 159 159 159 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 90 90 90 158 158 158 F F F Max gen: 92, Max flush: 162 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 158 158 158 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 86 86 86 152 152 152 F F F 4 080-099 91 91 91 159 159 159 F F F 5 100-119 93 93 93 163 163 163 F F F 6 120-139 93 93 93 163 163 163 F F F 7 140-159 90 90 90 158 158 158 F F F 8 160-179 92 92 92 160 160 160 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 93 93 93 163 163 163 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 90 90 90 158 158 158 F F F 16 320-339 92 92 92 162 162 162 F F F 17 340-359 93 93 93 163 163 163 F F F 18 360-379 93 93 93 163 163 163 F F F 19 380-399 92 92 92 162 162 162 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 92 92 92 161 161 161 F F F 25 500-519 89 89 89 155 155 155 F F F 26 520-539 89 89 89 155 155 155 F F F 27 540-559 91 91 91 159 159 159 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 90 90 90 158 158 158 F F F Max gen: 93, Max flush: 163 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.32664587Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:07.370086455Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.370614347Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.402753832Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.403780912Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.403828053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.40384512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.403863087Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566} {"msg":"Crucible 10e11ec6-31b7-4872-a6cd-0d5f8b8c5e51 has session id: 357a7d65-2b1b-4d0c-99d8-bab570750a45","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.404465345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Upstairs opts: Upstairs UUID: 10e11ec6-31b7-4872-a6cd-0d5f8b8c5e51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.404499299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Crucible stats registered with UUID: 10e11ec6-31b7-4872-a6cd-0d5f8b8c5e51","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.404515256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:07.404528334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.404995507Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405032241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566} {"msg":"10e11ec6-31b7-4872-a6cd-0d5f8b8c5e51 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.40508704Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405113506Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"0","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.40514222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"0","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405157427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"1","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405173634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"1","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405193051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"2","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405207938Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"2","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405221016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405276025Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.40530702Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:49960)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405322197Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:45596)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405673002Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:38779)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.405723423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.406157472Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"1","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.406326641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"2","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.406579014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"0","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.406796114Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"1","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.40687414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"2","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407158277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"0","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407430067Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407541376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[0]R flush_numbers[0..12]: [158, 155, 159, 152, 159, 163, 163, 158, 160, 161, 161, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407764205Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[0]R generation[0..12]: [90, 89, 91, 86, 91, 93, 93, 90, 92, 92, 92, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407785881Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407801388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[1]R flush_numbers[0..12]: [158, 155, 159, 152, 159, 163, 163, 158, 160, 161, 161, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407815116Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[1]R generation[0..12]: [90, 89, 91, 86, 91, 93, 93, 90, 92, 92, 92, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407828413Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407842141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[2]R flush_numbers[0..12]: [158, 155, 159, 152, 159, 163, 163, 158, 160, 161, 161, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407855508Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[2]R generation[0..12]: [90, 89, 91, 86, 91, 93, 93, 90, 92, 92, 92, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407868636Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407882453Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407895081Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:20:07.407907958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407922966Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407940252Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"":"downstairs","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.40795362Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407965598Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"0","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407978596Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"1","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.407991313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"client":"2","":"downstairs","session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.408013679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.408030566Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"10e11ec6-31b7-4872-a6cd-0d5f8b8c5e51 is now active with session: 357a7d65-2b1b-4d0c-99d8-bab570750a45","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.408043584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.408055271Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566,"session_id":"357a7d65-2b1b-4d0c-99d8-bab570750a45"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:07.408067099Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 212 len 12288 data: 7 6 5 02/30 Write block 6 len 4096 data: 8 03/30 Write block 11 len 16384 data: 13 10 11 9 04/30 Read block 553 len 8192 05/30 Read block 386 len 4096 06/30 Write block 527 len 24576 data: 6 7 5 4 4 3 07/30 Write block 367 len 24576 data: 11 9 8 7 7 9 08/30 Flush 09/30 Write block 151 len 20480 data: 9 9 7 7 6 10/30 Read block 128 len 36864 11/30 Write block 344 len 20480 data: 11 8 8 7 7 12/30 Write block 2 len 40960 data: 6 6 7 7 9 9 9 13 13 14 13/30 Read block 250 len 28672 14/30 Read block 266 len 20480 15/30 Read block 523 len 8192 16/30 Read block 485 len 4096 17/30 Write block 308 len 12288 data: 7 6 7 18/30 Read block 159 len 8192 19/30 Read block 296 len 12288 20/30 Read block 514 len 40960 21/30 Read block 251 len 20480 22/30 Read block 170 len 20480 23/30 Read block 160 len 28672 24/30 Write block 386 len 24576 data: 7 7 6 8 9 9 25/30 Write block 70 len 40960 data: 16 15 17 15 14 9 9 7 7 7 26/30 Read block 443 len 4096 27/30 Read block 533 len 16384 28/30 Write block 114 len 8192 data: 9 10 29/30 Write block 12 len 32768 data: 11 12 10 10 8 8 8 8 30/30 Write block 369 len 40960 data: 9 8 8 10 8 8 10 11 8 8 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 5 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:07.428381851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 164 164 164 T T T 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 86 86 86 152 152 152 T T T 4 080-099 91 91 91 159 159 159 F F F 5 100-119 93 93 93 163 163 163 T T T 6 120-139 93 93 93 163 163 163 F F F 7 140-159 90 90 90 158 158 158 T T T 8 160-179 92 92 92 160 160 160 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 93 93 93 163 163 163 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 90 90 90 158 158 158 T T T 16 320-339 92 92 92 162 162 162 F F F 17 340-359 93 93 93 163 163 163 T T T 18 360-379 94 94 94 164 164 164 T T T 19 380-399 92 92 92 162 162 162 T T T 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 92 92 92 161 161 161 F F F 25 500-519 89 89 89 155 155 155 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 91 91 91 159 159 159 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 90 90 90 158 158 158 F F F Max gen: 94, Max flush: 164 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 165 165 165 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 95 95 95 165 165 165 F F F 4 080-099 91 91 91 159 159 159 F F F 5 100-119 95 95 95 165 165 165 F F F 6 120-139 93 93 93 163 163 163 F F F 7 140-159 95 95 95 165 165 165 F F F 8 160-179 92 92 92 160 160 160 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 93 93 93 163 163 163 F F F 14 280-299 89 89 89 155 155 155 F F F 15 300-319 95 95 95 165 165 165 F F F 16 320-339 92 92 92 162 162 162 F F F 17 340-359 95 95 95 165 165 165 F F F 18 360-379 95 95 95 165 165 165 F F F 19 380-399 95 95 95 165 165 165 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 92 92 92 161 161 161 F F F 25 500-519 89 89 89 155 155 155 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 91 91 91 159 159 159 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 90 90 90 158 158 158 F F F Max gen: 95, Max flush: 165 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.742933207Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.789819314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.790882767Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.790936937Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.790954813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.79097246Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574} {"msg":"Crucible d4dad058-e2bf-4f33-87cc-b07a16295e8b has session id: 2c11ddb6-d386-4c6c-9165-756099ca8781","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.791684958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Upstairs opts: Upstairs UUID: d4dad058-e2bf-4f33-87cc-b07a16295e8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.791719812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Crucible stats registered with UUID: d4dad058-e2bf-4f33-87cc-b07a16295e8b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.791741428Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:11.791754456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792372501Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792399196Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574} {"msg":"d4dad058-e2bf-4f33-87cc-b07a16295e8b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792481831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792500138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"0","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792526273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"0","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.79254082Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"1","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792565056Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"1","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792589981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"2","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792617426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"2","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792643291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792669726Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792697121Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"ds_connection connected from Ok(127.0.0.1:61929)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792735454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"ds_connection connected from Ok(127.0.0.1:49724)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792768628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"ds_connection connected from Ok(127.0.0.1:45881)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.792791314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.793504832Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"0","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.793688308Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"2","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.794204252Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"0","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.79443608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"2","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.79459715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"1","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.794915411Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795063753Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795229493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"1","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[0]R flush_numbers[0..12]: [165, 155, 159, 165, 159, 165, 163, 165, 160, 161, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795778241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[0]R generation[0..12]: [95, 89, 91, 95, 91, 95, 93, 95, 92, 92, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795806706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795822143Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[1]R flush_numbers[0..12]: [165, 155, 159, 165, 159, 165, 163, 165, 160, 161, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795836121Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[1]R generation[0..12]: [95, 89, 91, 95, 91, 95, 93, 95, 92, 92, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795855207Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795870014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[2]R flush_numbers[0..12]: [165, 155, 159, 165, 159, 165, 163, 165, 160, 161, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795883962Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[2]R generation[0..12]: [95, 89, 91, 95, 91, 95, 93, 95, 92, 92, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795897269Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795910847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"Wait for a query_work_queue command to finish before sending IO msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795930993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795944001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795957588Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795970296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"":"downstairs","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795982973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.795994591Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"0","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.796007469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"1","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.796020296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"client":"2","":"downstairs","session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.796033174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.796045202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"d4dad058-e2bf-4f33-87cc-b07a16295e8b is now active with session: 2c11ddb6-d386-4c6c-9165-756099ca8781","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.796059889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.796071767Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574,"session_id":"2c11ddb6-d386-4c6c-9165-756099ca8781"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:11.796083665Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 366 len 32768 data: 10 12 10 10 9 9 11 9 02/30 Write block 509 len 20480 data: 7 7 7 6 6 03/30 Write block 331 len 20480 data: 5 7 8 8 7 04/30 Read block 535 len 20480 05/30 Read block 577 len 36864 06/30 Read block 344 len 8192 07/30 Read block 310 len 40960 08/30 Write block 70 len 28672 data: 17 16 18 16 15 10 10 09/30 Write block 125 len 16384 data: 7 7 7 5 10/30 Read block 311 len 12288 11/30 Write block 69 len 32768 data: 14 18 17 19 17 16 11 11 12/30 Read block 397 len 4096 13/30 Write block 297 len 24576 data: 9 5 4 4 5 3 14/30 Read block 313 len 32768 15/30 Write block 172 len 20480 data: 11 10 11 10 9 16/30 Read block 456 len 20480 17/30 Read block 245 len 8192 18/30 Read block 132 len 4096 19/30 Read block 224 len 24576 20/30 Write block 113 len 40960 data: 8 10 11 10 9 9 11 15 11 9 21/30 Read block 156 len 4096 22/30 Write block 304 len 32768 data: 2 4 6 6 8 7 8 8 23/30 Read block 308 len 24576 24/30 Flush 25/30 Write block 269 len 28672 data: 5 5 5 7 5 4 4 26/30 Write block 461 len 4096 data: 7 27/30 Read block 382 len 20480 28/30 Write block 580 len 4096 data: 9 29/30 Read block 530 len 16384 30/30 Read block 103 len 40960 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:11.817430796Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 165 165 165 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 96 96 96 166 166 166 F F F 4 080-099 91 91 91 159 159 159 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 96 96 96 166 166 166 F F F 7 140-159 95 95 95 165 165 165 F F F 8 160-179 96 96 96 166 166 166 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 93 93 93 163 163 163 T T T 14 280-299 96 96 96 166 166 166 F F F 15 300-319 96 96 96 166 166 166 F F F 16 320-339 96 96 96 166 166 166 F F F 17 340-359 95 95 95 165 165 165 F F F 18 360-379 96 96 96 166 166 166 F F F 19 380-399 95 95 95 165 165 165 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 92 92 92 160 160 160 T T T 24 480-499 92 92 92 161 161 161 F F F 25 500-519 96 96 96 166 166 166 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 91 91 91 159 159 159 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 90 90 90 158 158 158 T T T Max gen: 96, Max flush: 166 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 165 165 165 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 96 96 96 166 166 166 F F F 4 080-099 91 91 91 159 159 159 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 96 96 96 166 166 166 F F F 7 140-159 95 95 95 165 165 165 F F F 8 160-179 96 96 96 166 166 166 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 93 93 93 163 163 163 F F F 13 260-279 97 97 97 167 167 167 F F F 14 280-299 96 96 96 166 166 166 F F F 15 300-319 96 96 96 166 166 166 F F F 16 320-339 96 96 96 166 166 166 F F F 17 340-359 95 95 95 165 165 165 F F F 18 360-379 96 96 96 166 166 166 F F F 19 380-399 95 95 95 165 165 165 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 F F F 23 460-479 97 97 97 167 167 167 F F F 24 480-499 92 92 92 161 161 161 F F F 25 500-519 96 96 96 166 166 166 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 91 91 91 159 159 159 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 97 97 97 167 167 167 F F F Max gen: 97, Max flush: 167 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.068059906Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.113571837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.114516892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.114560344Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.11458167Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.114596548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582} {"msg":"Crucible f8796e52-b881-471c-aa69-5e44401faf98 has session id: f0eef41f-8bc0-405b-b97b-5c99d0c54d0f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115193957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Upstairs opts: Upstairs UUID: f8796e52-b881-471c-aa69-5e44401faf98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115235789Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Crucible stats registered with UUID: f8796e52-b881-471c-aa69-5e44401faf98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115259045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:16.115272923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.11571711Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115749144Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582} {"msg":"f8796e52-b881-471c-aa69-5e44401faf98 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115809453Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115836308Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"0","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115863943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"0","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.11587822Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"1","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115892818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"1","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115906415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"2","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115924562Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"2","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115938529Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.115962265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.11598955Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63503)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.116036341Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34079)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.116065326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59103)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.116136793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.116822746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"2","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.11685209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"0","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.117548871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"2","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.117589814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"0","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.118236574Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.118391666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.118488808Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"1","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119160503Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"1","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[0]R flush_numbers[0..12]: [165, 155, 159, 166, 159, 166, 166, 165, 166, 161, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119704423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[0]R generation[0..12]: [95, 89, 91, 96, 91, 96, 96, 95, 96, 92, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119725949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119741156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[1]R flush_numbers[0..12]: [165, 155, 159, 166, 159, 166, 166, 165, 166, 161, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119755083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[1]R generation[0..12]: [95, 89, 91, 96, 91, 96, 96, 95, 96, 92, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119768351Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119782118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[2]R flush_numbers[0..12]: [165, 155, 159, 166, 159, 166, 166, 165, 166, 161, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119795706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[2]R generation[0..12]: [95, 89, 91, 96, 91, 96, 96, 95, 96, 92, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119808713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119822301Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119834948Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119860784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119885449Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119910945Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"":"downstairs","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119940679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.119965364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"0","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.11999115Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"1","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.120017215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"client":"2","":"downstairs","session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.12004161Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.120056667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"f8796e52-b881-471c-aa69-5e44401faf98 is now active with session: f0eef41f-8bc0-405b-b97b-5c99d0c54d0f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.120069285Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.120081193Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582,"session_id":"f0eef41f-8bc0-405b-b97b-5c99d0c54d0f"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:16.120093011Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 89 len 24576 data: 9 7 5 6 9 9 02/30 Write block 184 len 12288 data: 8 8 7 03/30 Write block 343 len 24576 data: 9 12 9 9 8 8 04/30 Write block 513 len 40960 data: 7 6 6 6 4 5 5 5 5 5 05/30 Write block 353 len 28672 data: 7 8 10 13 12 12 13 06/30 Write block 8 len 32768 data: 10 14 14 15 12 13 11 11 07/30 Read block 363 len 40960 08/30 Read block 255 len 28672 09/30 Flush 10/30 Write block 288 len 16384 data: 6 7 9 10 11/30 Write block 531 len 12288 data: 5 4 3 12/30 Read block 63 len 16384 13/30 Write block 545 len 4096 data: 9 14/30 Read block 323 len 28672 15/30 Read block 108 len 36864 16/30 Write block 476 len 24576 data: 10 9 10 8 9 8 17/30 Write block 243 len 40960 data: 6 5 5 7 8 8 7 6 8 8 18/30 Write block 148 len 12288 data: 9 8 8 19/30 Write block 371 len 8192 data: 10 12 20/30 Write block 334 len 32768 data: 9 8 7 5 6 9 7 7 21/30 Read block 534 len 20480 22/30 Write block 62 len 16384 data: 6 5 6 9 23/30 Write block 546 len 12288 data: 8 6 7 24/30 Read block 109 len 16384 25/30 Write block 2 len 32768 data: 7 7 8 8 10 10 11 15 26/30 Read block 448 len 4096 27/30 Read block 151 len 20480 28/30 Write block 316 len 28672 data: 8 8 10 9 11 9 8 29/30 Write block 450 len 40960 data: 9 8 7 10 9 10 10 10 9 8 30/30 Read block 111 len 16384 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 4 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 20 20 21 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:16.139786618Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 168 168 168 T T T 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 96 96 96 166 166 166 T T T 4 080-099 98 98 98 168 168 168 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 96 96 96 166 166 166 F F F 7 140-159 95 95 95 165 165 165 T T T 8 160-179 96 96 96 166 166 166 F F F 9 180-199 98 98 98 168 168 168 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 93 93 93 163 163 163 T T T 13 260-279 97 97 97 167 167 167 F F F 14 280-299 96 96 96 166 166 166 T T T 15 300-319 96 96 96 166 166 166 T T T 16 320-339 96 96 96 166 166 166 T T T 17 340-359 98 98 98 168 168 168 T T T 18 360-379 96 96 96 166 166 166 T T T 19 380-399 95 95 95 165 165 165 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 88 88 88 154 154 154 T T T 23 460-479 97 97 97 167 167 167 T T T 24 480-499 92 92 92 161 161 161 T T T 25 500-519 98 98 98 168 168 168 F F F 26 520-539 98 98 98 168 168 168 T T T 27 540-559 91 91 91 159 159 159 T T T 28 560-579 90 90 90 158 158 158 F F F 29 580-599 97 97 97 167 167 167 F F F Max gen: 98, Max flush: 168 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 169 169 169 F F F 1 020-039 89 89 89 155 155 155 F F F 2 040-059 91 91 91 159 159 159 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 98 98 98 168 168 168 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 96 96 96 166 166 166 F F F 7 140-159 99 99 99 169 169 169 F F F 8 160-179 96 96 96 166 166 166 F F F 9 180-199 98 98 98 168 168 168 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 99 99 99 169 169 169 F F F 13 260-279 97 97 97 167 167 167 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 99 99 99 169 169 169 F F F 16 320-339 99 99 99 169 169 169 F F F 17 340-359 99 99 99 169 169 169 F F F 18 360-379 99 99 99 169 169 169 F F F 19 380-399 95 95 95 165 165 165 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 98 98 98 168 168 168 F F F 26 520-539 99 99 99 169 169 169 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 90 90 90 158 158 158 F F F 29 580-599 97 97 97 167 167 167 F F F Max gen: 99, Max flush: 169 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.448253229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.494674573Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.495757782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.495814622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.495832838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.495850765Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590} {"msg":"Crucible 32e6ee80-9f0b-4036-b63c-bcc71d11802c has session id: f84895aa-7327-4dd9-ac81-9451aecb632a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.496373508Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Upstairs opts: Upstairs UUID: 32e6ee80-9f0b-4036-b63c-bcc71d11802c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.496422329Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Crucible stats registered with UUID: 32e6ee80-9f0b-4036-b63c-bcc71d11802c","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.496449324Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:30.49647126Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497031826Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497057452Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590} {"msg":"32e6ee80-9f0b-4036-b63c-bcc71d11802c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497135177Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497152744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"0","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497178389Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"0","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497211453Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"1","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.49722719Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"1","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497240168Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497253845Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497266643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.49727996Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497293358Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58485)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497436382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51721)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497476304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:30.497512997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:30.497541572Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:30.497666989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.497852235Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.498284685Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"1","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.498325747Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"0","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.49895652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"0","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.49906791Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"1","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.499521186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:30.499676047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.498498275Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44375)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.498641928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.499886318Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.50058134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[0]R flush_numbers[0..12]: [169, 155, 159, 169, 168, 166, 166, 169, 166, 168, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501239728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[0]R generation[0..12]: [99, 89, 91, 99, 98, 96, 96, 99, 96, 98, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501270042Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501286749Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[1]R flush_numbers[0..12]: [169, 155, 159, 169, 168, 166, 166, 169, 166, 168, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501301047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[1]R generation[0..12]: [99, 89, 91, 99, 98, 96, 96, 99, 96, 98, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501314714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501328462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[2]R flush_numbers[0..12]: [169, 155, 159, 169, 168, 166, 166, 169, 166, 168, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501341999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[2]R generation[0..12]: [99, 89, 91, 99, 98, 96, 96, 99, 96, 98, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501355366Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501368784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501381382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501394609Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501407057Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501419505Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"":"downstairs","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501441011Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501459407Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"0","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501473145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"1","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501486322Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"client":"2","":"downstairs","session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.50149929Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501511488Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"32e6ee80-9f0b-4036-b63c-bcc71d11802c is now active with session: f84895aa-7327-4dd9-ac81-9451aecb632a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501523985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501535903Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590,"session_id":"f84895aa-7327-4dd9-ac81-9451aecb632a"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:40.501547941Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 39 len 32768 data: 5 8 6 6 4 6 7 7 02/30 Write block 561 len 40960 data: 10 8 8 9 10 8 8 8 5 5 03/30 Read block 107 len 8192 04/30 Write block 249 len 28672 data: 8 7 9 9 9 11 12 05/30 Write block 563 len 24576 data: 9 10 11 9 9 9 06/30 Write block 266 len 24576 data: 9 9 6 6 6 6 07/30 Read block 80 len 28672 08/30 Flush 09/30 Write block 581 len 20480 data: 9 8 6 6 5 10/30 Read block 463 len 28672 11/30 Write block 9 len 36864 data: 16 15 16 13 14 12 12 9 9 12/30 Flush 13/30 Write block 123 len 28672 data: 7 8 8 8 8 6 5 14/30 Read block 591 len 4096 15/30 Read block 329 len 40960 16/30 Write block 192 len 12288 data: 10 10 12 17/30 Read block 412 len 40960 18/30 Read block 401 len 28672 19/30 Write block 523 len 16384 data: 6 8 8 6 20/30 Write block 503 len 8192 data: 8 8 21/30 Write block 248 len 12288 data: 9 9 8 22/30 Write block 81 len 20480 data: 8 10 8 10 11 23/30 Write block 569 len 4096 data: 6 24/30 Read block 96 len 28672 25/30 Read block 558 len 36864 26/30 Flush 27/30 Write block 174 len 20480 data: 12 11 10 11 10 28/30 Read block 2 len 20480 29/30 Write block 18 len 28672 data: 9 9 9 10 8 10 9 30/30 Read block 492 len 16384 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:40.52290655Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 171 171 171 T T T 1 020-039 100 100 100 170 170 170 T T T 2 040-059 100 100 100 170 170 170 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 99 99 99 169 169 169 F F F 8 160-179 96 96 96 166 166 166 T T T 9 180-199 100 100 100 172 172 172 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 100 100 100 170 170 170 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 99 99 99 169 169 169 F F F 16 320-339 99 99 99 169 169 169 F F F 17 340-359 99 99 99 169 169 169 F F F 18 360-379 99 99 99 169 169 169 F F F 19 380-399 95 95 95 165 165 165 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 100 100 100 172 172 172 F F F 26 520-539 100 100 100 172 172 172 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 100 100 172 172 172 F F F 29 580-599 100 100 100 171 171 171 F F F Max gen: 100, Max flush: 172 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 173 173 173 F F F 1 020-039 101 101 101 173 173 173 F F F 2 040-059 100 100 100 170 170 170 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 99 99 99 169 169 169 F F F 8 160-179 101 101 101 173 173 173 F F F 9 180-199 100 100 100 172 172 172 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 100 100 100 170 170 170 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 99 99 99 169 169 169 F F F 16 320-339 99 99 99 169 169 169 F F F 17 340-359 99 99 99 169 169 169 F F F 18 360-379 99 99 99 169 169 169 F F F 19 380-399 95 95 95 165 165 165 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 89 89 89 155 155 155 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 100 100 100 172 172 172 F F F 26 520-539 100 100 100 172 172 172 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 100 100 172 172 172 F F F 29 580-599 100 100 100 171 171 171 F F F Max gen: 101, Max flush: 173 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.764305327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.810170653Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.811208811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.811252983Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.81126806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.811282667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598} {"msg":"Crucible 69710e86-71ba-4ac7-ab3a-0926d6fab396 has session id: 453cc9d1-a897-4d5c-b12f-69c96429634e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.811895594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Upstairs opts: Upstairs UUID: 69710e86-71ba-4ac7-ab3a-0926d6fab396, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.811939146Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Crucible stats registered with UUID: 69710e86-71ba-4ac7-ab3a-0926d6fab396","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.811962851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:44.811981978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812421107Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812447152Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598} {"msg":"69710e86-71ba-4ac7-ab3a-0926d6fab396 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812503941Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812530376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"0","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.81256363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"0","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812579577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812594195Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812607502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"2","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.81262141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"2","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812634037Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812647535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812660792Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62647)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.812814424Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56986)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.813056209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:44.81331911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:44.813353494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:44.813371281Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.813481051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"0","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.813522623Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.813561696Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"2","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.81413245Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"0","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.814273014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"2","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.81472799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:44.814840599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.815064717Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39983)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.815193194Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.816204147Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.816752275Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[0]R flush_numbers[0..12]: [173, 173, 170, 169, 172, 166, 172, 169, 173, 172, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.81737313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[0]R generation[0..12]: [101, 101, 100, 99, 100, 96, 100, 99, 101, 100, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817399086Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817414513Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[1]R flush_numbers[0..12]: [173, 173, 170, 169, 172, 166, 172, 169, 173, 172, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.81742896Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[1]R generation[0..12]: [101, 101, 100, 99, 100, 96, 100, 99, 101, 100, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817449316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817464063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[2]R flush_numbers[0..12]: [173, 173, 170, 169, 172, 166, 172, 169, 173, 172, 164, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817478061Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[2]R generation[0..12]: [101, 101, 100, 99, 100, 96, 100, 99, 101, 100, 94, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817491318Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817504826Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817517394Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817530251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817542689Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817555217Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"":"downstairs","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817567904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817579522Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"0",""Wait for a query_work_queue command to finish before sending IO :"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817602788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"1","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817615546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"client":"2","":"downstairs","session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817628263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817640061Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"69710e86-71ba-4ac7-ab3a-0926d6fab396 is now active with session: 453cc9d1-a897-4d5c-b12f-69c96429634e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817654678Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817666516Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598,"session_id":"453cc9d1-a897-4d5c-b12f-69c96429634e"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:54.817678324Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 357 len 36864 data: 13 13 14 12 11 9 8 9 9 02/30 Write block 265 len 12288 data: 9 10 10 03/30 Write block 432 len 24576 data: 5 6 8 8 10 10 04/30 Read block 578 len 24576 05/30 Write block 179 len 32768 data: 12 12 9 10 8 9 9 8 06/30 Flush 07/30 Write block 230 len 12288 data: 8 9 11 08/30 Write block 558 len 36864 data: 7 8 9 11 9 10 11 12 10 09/30 Write block 310 len 40960 data: 9 9 10 10 7 6 9 9 11 10 10/30 Flush 11/30 Write block 367 len 32768 data: 13 11 11 10 11 13 10 9 12/30 Write block 332 len 40960 data: 8 9 10 9 8 6 7 10 8 8 13/30 Read block 403 len 16384 14/30 Read block 595 len 12288 15/30 Read block 112 len 32768 16/30 Write block 397 len 28672 data: 4 6 7 9 10 8 8 17/30 Read block 553 len 16384 18/30 Read block 280 len 40960 19/30 Write block 597 len 8192 data: 3 3 20/30 Write block 408 len 12288 data: 6 6 7 21/30 Read block 134 len 12288 22/30 Write block 394 len 4096 data: 8 23/30 Read block 391 len 28672 24/30 Write block 375 len 40960 data: 11 12 9 9 8 8 7 6 7 7 25/30 Read block 203 len 32768 26/30 Read block 587 len 4096 27/30 Write block 1 len 40960 data: 5 8 8 9 9 11 11 12 17 16 28/30 Write block 235 len 20480 data: 7 8 7 8 9 29/30 Read block 256 len 32768 30/30 Read block 499 len 24576 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 8 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 20 19 20 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:54.840209016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 173 173 173 T T T 1 020-039 101 101 101 173 173 173 F F F 2 040-059 100 100 100 170 170 170 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 99 99 99 169 169 169 F F F 8 160-179 102 102 102 174 174 174 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 102 102 102 175 175 175 T T T 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 99 99 99 169 169 169 T T T 17 340-359 102 102 102 174 174 174 T T T 18 360-379 102 102 102 174 174 174 T T T 19 380-399 95 95 95 165 165 165 T T T 20 400-419 92 92 92 162 162 162 T T T 21 420-439 102 102 102 174 174 174 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 100 100 100 172 172 172 F F F 26 520-539 100 100 100 172 172 172 F F F 27 540-559 102 102 102 175 175 175 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 100 100 100 171 171 171 T T T Max gen: 102, Max flush: 175 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 176 176 176 F F F 1 020-039 101 101 101 173 173 173 F F F 2 040-059 100 100 100 170 170 170 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 99 99 99 169 169 169 F F F 8 160-179 102 102 102 174 174 174 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 103 103 103 176 176 176 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 103 103 103 176 176 176 F F F 18 360-379 103 103 103 176 176 176 F F F 19 380-399 103 103 103 176 176 176 F F F 20 400-419 103 103 103 176 176 176 F F F 21 420-439 102 102 102 174 174 174 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 100 100 100 172 172 172 F F F 26 520-539 100 100 100 172 172 172 F F F 27 540-559 102 102 102 175 175 175 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 103 103 103 176 176 176 F F F Max gen: 103, Max flush: 176 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.111884543Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.158619858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.159685471Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.159739791Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.159757768Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.159775414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606} {"msg":"Crucible bdc1da7c-2a99-4516-ac34-b5e5e9e8b173 has session id: f0905f83-489d-49ae-8660-6d57d70741b9","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.160512478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Upstairs opts: Upstairs UUID: bdc1da7c-2a99-4516-ac34-b5e5e9e8b173, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.16055327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Crucible stats registered with UUID: bdc1da7c-2a99-4516-ac34-b5e5e9e8b173","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.160570457Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:59.160592873Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161195982Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161236084Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606} {"msg":"bdc1da7c-2a99-4516-ac34-b5e5e9e8b173 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161305611Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161332226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"0","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161360771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"0","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161395114Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"1","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161424449Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"1","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161449904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"2","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.1614751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"2","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161502045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.16152587Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161550356Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:49948)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161585349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58645)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.161608625Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54255)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.16163608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.162308705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"0","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.16233695Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"2","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.162977402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"0","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.163014985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"2","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.163535069Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"1","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.163701878Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.163726533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164052443Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"1","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[0]R flush_numbers[0..12]: [176, 173, 170, 169, 172, 166, 172, 169, 174, 174, 164, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164602321Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[0]R generation[0..12]: [103, 101, 100, 99, 100, 96, 100, 99, 102, 102, 94, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164624287Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164639074Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[1]R flush_numbers[0..12]: [176, 173, 170, 169, 172, 166, 172, 169, 174, 174, 164, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164653072Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[1]R generation[0..12]: [103, 101, 100, 99, 100, 96, 100, 99, 102, 102, 94, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164666849Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164680667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[2]R flush_numbers[0..12]: [176, 173, 170, 169, 172, 166, 172, 169, 174, 174, 164, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164694244Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[2]R generation[0..12]: [103, 101, 100, 99, 100, 96, 100, 99, 102, 102, 94, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164707882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164721699Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164744195Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.16477428Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164800655Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.16482551Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"":"downstairs","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164849366Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:20:59.164870652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"0","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164896597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"1","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164921332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"client":"2","":"downstairs","session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164937369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164949567Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"bdc1da7c-2a99-4516-ac34-b5e5e9e8b173 is now active with session: f0905f83-489d-49ae-8660-6d57d70741b9","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164962365Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.164973952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606,"session_id":"f0905f83-489d-49ae-8660-6d57d70741b9"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:20:59.1649858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 374 len 36864 data: 10 12 13 10 10 9 9 8 7 02/30 Write block 374 len 24576 data: 11 13 14 11 11 10 03/30 Flush 04/30 Flush 05/30 Write block 303 len 24576 data: 2 3 5 7 7 9 06/30 Write block 344 len 16384 data: 13 10 10 9 07/30 Read block 296 len 16384 08/30 Read block 85 len 24576 09/30 Read block 418 len 24576 10/30 Flush 11/30 Flush 12/30 Write block 146 len 16384 data: 11 10 10 9 13/30 Write block 529 len 24576 data: 6 5 6 5 4 3 14/30 Read block 153 len 12288 15/30 Write block 384 len 32768 data: 8 6 8 8 7 9 10 10 16/30 Write block 356 len 40960 data: 14 14 14 15 13 12 10 9 10 10 17/30 Read block 183 len 36864 18/30 Read block 344 len 4096 19/30 Read block 199 len 12288 20/30 Read block 149 len 24576 21/30 Write block 15 len 16384 data: 13 10 10 10 22/30 Flush 23/30 Write block 457 len 36864 data: 11 10 9 9 8 8 8 8 7 24/30 Write block 520 len 32768 data: 6 6 6 7 9 9 7 7 25/30 Read block 309 len 28672 26/30 Flush 27/30 Write block 222 len 40960 data: 5 5 4 3 4 5 7 7 9 10 28/30 Write block 406 len 20480 data: 11 9 7 7 8 29/30 Read block 294 len 4096 30/30 Read block 197 len 8192 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:20:59.186637895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 101 101 101 173 173 173 F F F 2 040-059 100 100 100 170 170 170 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 104 104 104 181 181 181 F F F 8 160-179 102 102 102 174 174 174 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 103 103 103 176 176 176 T T T 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 104 104 104 179 179 179 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 181 181 181 F F F 18 360-379 104 104 104 181 181 181 F F F 19 380-399 104 104 104 181 181 181 F F F 20 400-419 103 103 103 176 176 176 T T T 21 420-439 102 102 102 174 174 174 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 104 104 104 182 182 182 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 100 100 100 172 172 172 F F F 26 520-539 104 104 104 182 182 182 F F F 27 540-559 102 102 102 175 175 175 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 103 103 103 176 176 176 F F F Max gen: 104, Max flush: 182 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 101 101 101 173 173 173 F F F 2 040-059 100 100 100 170 170 170 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 104 104 104 181 181 181 F F F 8 160-179 102 102 102 174 174 174 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 105 105 105 183 183 183 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 104 104 104 179 179 179 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 181 181 181 F F F 18 360-379 104 104 104 181 181 181 F F F 19 380-399 104 104 104 181 181 181 F F F 20 400-419 105 105 105 183 183 183 F F F 21 420-439 102 102 102 174 174 174 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 104 104 104 182 182 182 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 100 100 100 172 172 172 F F F 26 520-539 104 104 104 182 182 182 F F F 27 540-559 102 102 102 175 175 175 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 103 103 103 176 176 176 F F F Max gen: 105, Max flush: 183 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.433085889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.481309688Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.482310913Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.482357075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.482372722Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.482387119Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614} {"msg":"Crucible c9f60963-0d28-4f52-9d6f-526c6990ed62 has session id: 32347c39-de66-4ccd-8fa7-f3b403f74a84","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483058485Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Upstairs opts: Upstairs UUID: c9f60963-0d28-4f52-9d6f-526c6990ed62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483109626Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Crucible stats registered with UUID: c9f60963-0d28-4f52-9d6f-526c6990ed62","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.48313909Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:03.483164855Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483745068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483769764Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614} {"msg":"c9f60963-0d28-4f52-9d6f-526c6990ed62 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483832912Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483859477Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"0","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483886712Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"0","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483901219Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"1","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483915677Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"1","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483929244Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"2","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483943311Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"2","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.483956109Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.484007809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.484027666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"ds_connection connected from Ok(127.0.0.1:40525)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.484128567Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"ds_connection connected from Ok(127.0.0.1:58676)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.48416424Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"ds_connection connected from Ok(127.0.0.1:56166)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.484266412Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.484899024Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"2","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.485057405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"0","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.485531028Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"2","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.485689628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"0","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.486143094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.486242906Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"1","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.48627639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.486819689Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"1","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[0]R flush_numbers[0..12]: [181, 173, 170, 169, 172, 166, 172, 181, 174, 174, 164, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.48735337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[0]R generation[0..12]: [104, 101, 100, 99, 100, 96, 100, 104, 102, 102, 94, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487378816Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487394453Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[1]R flush_numbers[0..12]: [181, 173, 170, 169, 172, 166, 172, 181, 174, 174, 164, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.4874084Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[1]R generation[0..12]: [104, 101, 100, 99, 100, 96, 100, 104, 102, 102, 94, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487422038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487435895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[2]R flush_numbers[0..12]: [181, 173, 170, 169, 172, 166, 172, 181, 174, 174, 164, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487470509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[2]R generation[0..12]: [104, 101, 100, 99, 100, 96, 100, 104, 102, 102, 94, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487485866Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487499714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:21:03.487512371Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487529788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487542426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487555193Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"":"downstairs","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487567971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487579719Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"0","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487592776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"1","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487605814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"client":"2","":"downstairs","session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487618641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487630589Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"c9f60963-0d28-4f52-9d6f-526c6990ed62 is now active with session: 32347c39-de66-4ccd-8fa7-f3b403f74a84","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487643247Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487655025Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614,"session_id":"32347c39-de66-4ccd-8fa7-f3b403f74a84"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:03.487670052Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 12288 data: 7 5 7 02/30 Read block 559 len 32768 03/30 Read block 132 len 8192 04/30 Write block 153 len 12288 data: 8 8 8 05/30 Write block 498 len 24576 data: 8 7 8 4 6 9 06/30 Read block 365 len 20480 07/30 Flush 08/30 Write block 499 len 4096 data: 8 09/30 Flush 10/30 Write block 505 len 12288 data: 10 9 8 11/30 Read block 29 len 4096 12/30 Flush 13/30 Write block 67 len 32768 data: 8 10 15 19 18 20 18 17 14/30 Write block 462 len 32768 data: 9 9 9 8 6 7 7 9 15/30 Flush 16/30 Write block 21 len 24576 data: 11 9 11 10 11 7 17/30 Write block 413 len 36864 data: 7 8 7 7 7 7 9 9 7 18/30 Read block 308 len 36864 19/30 Read block 243 len 20480 20/30 Read block 200 len 28672 21/30 Write block 412 len 28672 data: 8 8 9 8 8 8 8 22/30 Read block 355 len 8192 23/30 Flush 24/30 Write block 71 len 20480 data: 19 21 19 18 12 25/30 Read block 463 len 4096 26/30 Write block 348 len 24576 data: 9 9 7 6 6 8 27/30 Read block 196 len 36864 28/30 Write block 424 len 12288 data: 11 7 8 29/30 Write block 147 len 8192 data: 11 11 30/30 Write block 156 len 12288 data: 6 8 8 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:03.507283094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 106 106 106 188 188 188 F F F 2 040-059 100 100 100 170 170 170 F F F 3 060-079 106 106 106 187 187 187 T T T 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 106 106 106 184 184 184 T T T 8 160-179 102 102 102 174 174 174 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 105 105 105 183 183 183 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 104 104 104 179 179 179 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 181 181 181 T T T 18 360-379 104 104 104 181 181 181 F F F 19 380-399 104 104 104 181 181 181 F F F 20 400-419 106 106 106 188 188 188 F F F 21 420-439 106 106 106 188 188 188 T T T 22 440-459 104 104 104 182 182 182 F F F 23 460-479 106 106 106 187 187 187 F F F 24 480-499 106 106 106 185 185 185 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 104 104 104 182 182 182 F F F 27 540-559 102 102 102 175 175 175 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 103 103 103 176 176 176 F F F Max gen: 106, Max flush: 188 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 106 106 106 188 188 188 F F F 2 040-059 100 100 100 170 170 170 F F F 3 060-079 107 107 107 189 189 189 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 107 107 107 189 189 189 F F F 8 160-179 102 102 102 174 174 174 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 94 94 94 164 164 164 F F F 11 220-239 105 105 105 183 183 183 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 104 104 104 179 179 179 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 107 107 107 189 189 189 F F F 18 360-379 104 104 104 181 181 181 F F F 19 380-399 104 104 104 181 181 181 F F F 20 400-419 106 106 106 188 188 188 F F F 21 420-439 107 107 107 189 189 189 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 106 106 106 187 187 187 F F F 24 480-499 106 106 106 185 185 185 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 104 104 104 182 182 182 F F F 27 540-559 102 102 102 175 175 175 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 103 103 103 176 176 176 F F F Max gen: 107, Max flush: 189 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.756719325Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.802982326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.804024993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.804070385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.804085702Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.804099879Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622} {"msg":"Crucible 3b1e02fe-6e2e-4e36-a775-8adcb00bb3df has session id: 76318693-89ef-4da8-a6a5-bf544c46c57e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.804826285Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Upstairs opts: Upstairs UUID: 3b1e02fe-6e2e-4e36-a775-8adcb00bb3df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.804880005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Crucible stats registered with UUID: 3b1e02fe-6e2e-4e36-a775-8adcb00bb3df","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.804895542Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:07.80490809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805480884Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805505849Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622} {"msg":"3b1e02fe-6e2e-4e36-a775-8adcb00bb3df active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805586634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805604571Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"0","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805630916Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"0","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805651012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"1","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805677197Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"1","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805702363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805729108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805747635Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805766661Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805780378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43894)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.805949637Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:07.805992049Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:07.806025103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:07.806055707Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46993)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.806079743Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.806249662Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.806702768Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"0","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.806730013Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"1","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.807459778Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"1","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.807498001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"0","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.808168617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:07.808202081Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.807156715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56775)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.807264544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.808545727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809154665Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[0]R flush_numbers[0..12]: [181, 188, 170, 189, 172, 166, 172, 189, 174, 174, 164, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809803135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[0]R generation[0..12]: [104, 106, 100, 107, 100, 96, 100, 107, 102, 102, 94, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.80983059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809846307Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[1]R flush_numbers[0..12]: [181, 188, 170, 189, 172, 166, 172, 189, 174, 174, 164, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809860375Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[1]R generation[0..12]: [104, 106, 100, 107, 100, 96, 100, 107, 102, 102, 94, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809873752Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809887659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[2]R flush_numbers[0..12]: [181, 188, 170, 189, 172, 166, 172, 189, 174, 174, 164, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809901177Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[2]R generation[0..12]: [104, 106, 100, 107, 100, 96, 100, 107, 102, 102, 94, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809914574Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809928132Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.8099407Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809962145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.809975823Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-07T18:21:17.809996419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"":"downstairs","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810010087Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810021745Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"0","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810034632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"1","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.81004774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"client":"2","":"downstairs","session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810060707Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810072575Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"3b1e02fe-6e2e-4e36-a775-8adcb00bb3df is now active with session: 76318693-89ef-4da8-a6a5-bf544c46c57e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810085023Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810096611Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622,"session_id":"76318693-89ef-4da8-a6a5-bf544c46c57e"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:17.810108578Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 423 len 8192 data: 8 12 02/30 Write block 207 len 4096 data: 10 03/30 Write block 536 len 40960 data: 3 6 7 8 8 8 11 10 10 10 04/30 Read block 272 len 12288 05/30 Read block 570 len 24576 06/30 Read block 129 len 16384 07/30 Write block 176 len 4096 data: 11 08/30 Read block 18 len 36864 09/30 Read block 229 len 36864 10/30 Write block 376 len 16384 data: 15 12 12 11 11/30 Write block 559 len 40960 data: 9 10 12 10 11 12 13 11 10 10 12/30 Read block 45 len 8192 13/30 Write block 59 len 28672 data: 4 6 6 7 6 7 10 14/30 Read block 72 len 32768 15/30 Read block 595 len 12288 16/30 Read block 570 len 40960 17/30 Read block 288 len 32768 18/30 Read block 191 len 40960 19/30 Read block 270 len 20480 20/30 Read block 465 len 40960 21/30 Write block 333 len 32768 data: 10 11 10 9 7 8 11 9 22/30 Read block 444 len 40960 23/30 Write block 399 len 36864 data: 8 10 11 9 9 9 11 12 10 24/30 Read block 56 len 4096 25/30 Read block 355 len 28672 26/30 Flush 27/30 Write block 327 len 24576 data: 10 8 6 5 6 9 28/30 Read block 367 len 20480 29/30 Write block 581 len 16384 data: 10 9 7 7 30/30 Read block 275 len 8192 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:17.83280608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 106 106 106 188 188 188 F F F 2 040-059 108 108 108 190 190 190 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 107 107 107 189 189 189 F F F 8 160-179 108 108 108 190 190 190 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 105 105 105 183 183 183 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 104 104 104 179 179 179 F F F 16 320-339 108 108 108 190 190 190 T T T 17 340-359 108 108 108 190 190 190 F F F 18 360-379 108 108 108 190 190 190 F F F 19 380-399 108 108 108 190 190 190 F F F 20 400-419 108 108 108 190 190 190 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 106 106 106 187 187 187 F F F 24 480-499 106 106 106 185 185 185 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 108 108 108 190 190 190 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 108 108 108 190 190 190 F F F 29 580-599 103 103 103 176 176 176 T T T Max gen: 108, Max flush: 190 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 106 106 106 188 188 188 F F F 2 040-059 108 108 108 190 190 190 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 96 96 96 166 166 166 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 107 107 107 189 189 189 F F F 8 160-179 108 108 108 190 190 190 F F F 9 180-199 102 102 102 174 174 174 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 105 105 105 183 183 183 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 99 99 99 169 169 169 F F F 15 300-319 104 104 104 179 179 179 F F F 16 320-339 109 109 109 191 191 191 F F F 17 340-359 108 108 108 190 190 190 F F F 18 360-379 108 108 108 190 190 190 F F F 19 380-399 108 108 108 190 190 190 F F F 20 400-419 108 108 108 190 190 190 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 106 106 106 187 187 187 F F F 24 480-499 106 106 106 185 185 185 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 108 108 108 190 190 190 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 108 108 108 190 190 190 F F F 29 580-599 109 109 109 191 191 191 F F F Max gen: 109, Max flush: 191 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.072853207Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.120448763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.121537622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.121583083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.12159817Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.121612548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630} {"msg":"Crucible f8c9d7bf-05fb-4883-9efd-a0aa27aee6d1 has session id: 25d1c305-e566-480e-8382-3a21fade5213","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122102837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Upstairs opts: Upstairs UUID: f8c9d7bf-05fb-4883-9efd-a0aa27aee6d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.1221442Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Crucible stats registered with UUID: f8c9d7bf-05fb-4883-9efd-a0aa27aee6d1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122166985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:22.122187401Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122671532Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122697307Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630} {"msg":"f8c9d7bf-05fb-4883-9efd-a0aa27aee6d1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122762155Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122779262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"0","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122807887Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"0","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122826363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"1","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122852689Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"1","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122877924Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"2","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122954859Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"2","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.122982294Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.12300726Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.123029346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"ds_connection connected from Ok(127.0.0.1:36306)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.123052241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"ds_connection connected from Ok(127.0.0.1:61630)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.123217651Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"ds_connection connected from Ok(127.0.0.1:44979)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.123253894Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.123954754Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"1","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.123991948Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"0","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.124447134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"2","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.124620581Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"1","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.124766794Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"0","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125096653Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"2","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125345657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125465585Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[0]R flush_numbers[0..12]: [181, 188, 190, 190, 172, 166, 172, 189, 190, 174, 190, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125770669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[0]R generation[0..12]: [104, 106, 108, 108, 100, 96, 100, 107, 108, 102, 108, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125790295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125804782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[1]R flush_numbers[0..12]: [181, 188, 190, 190, 172, 166, 172, 189, 190, 174, 190, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.12581863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[1]R generation[0..12]: [104, 106, 108, 108, 100, 96, 100, 107, 108, 102, 108, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125839816Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125864681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[2]R flush_numbers[0..12]: [181, 188, 190, 190, 172, 166, 172, 189, 190, 174, 190, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125897155Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[2]R generation[0..12]: [104, 106, 108, 108, 100, 96, 100, 107, 108, 102, 108, 105]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:21:22.1259251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.125952525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.12597853Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126020263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126041679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126064794Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"":"downstairs","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.12608989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126112266Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"0","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126137761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"1","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126162946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"client":"2","":"downstairs","session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126181703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.12619399Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"f8c9d7bf-05fb-4883-9efd-a0aa27aee6d1 is now active with session: 25d1c305-e566-480e-8382-3a21fade5213","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126214947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126240672Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630,"session_id":"25d1c305-e566-480e-8382-3a21fade5213"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:22.126264707Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 4096 data: 7 02/30 Read block 252 len 8192 03/30 Flush 04/30 Write block 110 len 28672 data: 9 6 6 9 11 12 11 05/30 Write block 344 len 16384 data: 14 11 11 10 06/30 Write block 175 len 32768 data: 12 12 12 11 13 13 10 11 07/30 Write block 233 len 24576 data: 11 6 8 9 8 9 08/30 Read block 382 len 28672 09/30 Write block 38 len 16384 data: 7 6 9 7 10/30 Read block 41 len 36864 11/30 Read block 103 len 12288 12/30 Read block 469 len 20480 13/30 Flush 14/30 Write block 319 len 36864 data: 11 12 10 9 9 9 10 11 11 15/30 Read block 324 len 40960 16/30 Read block 127 len 24576 17/30 Flush 18/30 Write block 280 len 24576 data: 8 6 7 6 8 5 19/30 Read block 385 len 24576 20/30 Read block 61 len 28672 21/30 Flush 22/30 Write block 157 len 4096 data: 9 23/30 Read block 356 len 4096 24/30 Read block 472 len 4096 25/30 Read block 452 len 40960 26/30 Read block 128 len 24576 27/30 Read block 386 len 32768 28/30 Read block 50 len 8192 29/30 Read block 78 len 8192 30/30 Write block 21 len 12288 data: 12 10 12 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:22.146671332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 110 110 110 193 193 193 T T T 2 040-059 110 110 110 193 193 193 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 110 110 110 193 193 193 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 107 107 107 189 189 189 T T T 8 160-179 110 110 110 193 193 193 F F F 9 180-199 110 110 110 193 193 193 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 110 110 110 193 193 193 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 110 110 110 195 195 195 F F F 15 300-319 110 110 110 194 194 194 F F F 16 320-339 110 110 110 194 194 194 F F F 17 340-359 110 110 110 193 193 193 F F F 18 360-379 108 108 108 190 190 190 F F F 19 380-399 108 108 108 190 190 190 F F F 20 400-419 108 108 108 190 190 190 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 106 106 106 187 187 187 F F F 24 480-499 106 106 106 185 185 185 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 108 108 108 190 190 190 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 108 108 108 190 190 190 F F F 29 580-599 109 109 109 191 191 191 F F F Max gen: 110, Max flush: 195 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 111 111 111 196 196 196 F F F 2 040-059 110 110 110 193 193 193 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 110 110 110 193 193 193 F F F 6 120-139 100 100 100 172 172 172 F F F 7 140-159 111 111 111 196 196 196 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 110 110 110 193 193 193 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 110 110 110 193 193 193 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 F F F 14 280-299 110 110 110 195 195 195 F F F 15 300-319 110 110 110 194 194 194 F F F 16 320-339 110 110 110 194 194 194 F F F 17 340-359 110 110 110 193 193 193 F F F 18 360-379 108 108 108 190 190 190 F F F 19 380-399 108 108 108 190 190 190 F F F 20 400-419 108 108 108 190 190 190 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 106 106 106 187 187 187 F F F 24 480-499 106 106 106 185 185 185 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 108 108 108 190 190 190 F F F 27 540-559 108 108 108 190 190 190 F F F 28 560-579 108 108 108 190 190 190 F F F 29 580-599 109 109 109 191 191 191 F F F Max gen: 111, Max flush: 196 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.391890133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.439436718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.440476686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.440522427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.440537394Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.440551462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638} {"msg":"Crucible 4d5d503c-fc22-43cb-b288-e3f41738b368 has session id: 75b57327-5fef-4150-9276-3b05757f5982","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441252872Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Upstairs opts: Upstairs UUID: 4d5d503c-fc22-43cb-b288-e3f41738b368, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441296264Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Crucible stats registered with UUID: 4d5d503c-fc22-43cb-b288-e3f41738b368","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441321829Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:26.441342635Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441820047Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441847972Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638} {"msg":"4d5d503c-fc22-43cb-b288-e3f41738b368 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441917569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441937525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"0","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.44196735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"0","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.441992855Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"1","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.44202012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"1","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442042206Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"2","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442067381Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"2","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442091747Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442121591Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442144937Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"ds_connection connected from Ok(127.0.0.1:58124)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442165993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"ds_connection connected from Ok(127.0.0.1:49886)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442189889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"ds_connection connected from Ok(127.0.0.1:64308)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442214324Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.442975053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"0","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.443014246Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"1","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.443667785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"1","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.44369775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"0","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.444093817Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"2","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.44428947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.444387092Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.444658922Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"2","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[0]R flush_numbers[0..12]: [181, 196, 193, 190, 172, 193, 172, 196, 193, 193, 190, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445232706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[0]R generation[0..12]: [104, 111, 110, 108, 100, 110, 100, 111, 110, 110, 108, 110]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445252752Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445272568Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[1]R flush_numbers[0..12]: [181, 196, 193, 190, 172, 193, 172, 196, 193, 193, 190, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445286676Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[1]R generation[0..12]: [104, 111, 110, 108, 100, 110, 100, 111, 110, 110, 108, 110]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445299963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445313741Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[2]R flush_numbers[0..12]: [181, 196, 193, 190, 172, 193, 172, 196, 193, 193, 190, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445327248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[2]R generation[0..12]: [104, 111, 110, 108, 100, 110, 100, 111, 110, 110, 108, 110]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445340686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445354323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445366911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445379539Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:21:26.445391877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445407663Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"":"downstairs","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445420431Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445432089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"0","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445445076Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"1","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445460884Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"client":"2","":"downstairs","session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445474201Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445486209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"4d5d503c-fc22-43cb-b288-e3f41738b368 is now active with session: 75b57327-5fef-4150-9276-3b05757f5982","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445498857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445510475Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638,"session_id":"75b57327-5fef-4150-9276-3b05757f5982"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:26.445522492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 570 len 24576 data: 6 4 5 8 9 8 02/30 Flush 03/30 Write block 413 len 8192 data: 9 10 04/30 Write block 127 len 20480 data: 9 7 6 4 3 05/30 Write block 558 len 12288 data: 8 10 11 06/30 Write block 382 len 20480 data: 8 8 9 7 9 07/30 Flush 08/30 Write block 524 len 8192 data: 10 10 09/30 Read block 324 len 20480 10/30 Read block 251 len 4096 11/30 Write block 542 len 40960 data: 12 11 11 11 9 7 8 5 6 5 12/30 Write block 45 len 28672 data: 8 8 6 7 11 8 8 13/30 Write block 553 len 12288 data: 7 6 6 14/30 Write block 479 len 40960 data: 9 10 9 9 9 12 11 12 11 12 15/30 Flush 16/30 Write block 272 len 36864 data: 8 6 5 5 5 7 7 8 9 17/30 Read block 350 len 8192 18/30 Read block 36 len 24576 19/30 Read block 483 len 28672 20/30 Write block 399 len 24576 data: 9 11 12 10 10 10 21/30 Write block 307 len 40960 data: 8 10 8 10 10 11 11 8 7 10 22/30 Write block 314 len 8192 data: 9 8 23/30 Write block 566 len 40960 data: 12 11 11 7 7 5 6 9 10 9 24/30 Write block 577 len 24576 data: 7 8 8 10 11 10 25/30 Read block 192 len 40960 26/30 Write block 123 len 8192 data: 8 9 27/30 Read block 101 len 32768 28/30 Write block 133 len 4096 data: 6 29/30 Write block 313 len 16384 data: 12 10 9 11 30/30 Write block 544 len 20480 data: 12 12 10 8 9 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:26.465030944Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 111 111 111 196 196 196 F F F 2 040-059 112 112 112 199 199 199 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 110 110 110 193 193 193 F F F 6 120-139 112 112 112 198 198 198 T T T 7 140-159 111 111 111 196 196 196 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 110 110 110 193 193 193 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 110 110 110 193 193 193 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 102 102 102 174 174 174 T T T 14 280-299 110 110 110 195 195 195 T T T 15 300-319 110 110 110 194 194 194 T T T 16 320-339 110 110 110 194 194 194 F F F 17 340-359 110 110 110 193 193 193 F F F 18 360-379 108 108 108 190 190 190 F F F 19 380-399 112 112 112 198 198 198 T T T 20 400-419 112 112 112 198 198 198 T T T 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 112 112 112 199 199 199 F F F 27 540-559 112 112 112 199 199 199 T T T 28 560-579 112 112 112 198 198 198 T T T 29 580-599 109 109 109 191 191 191 T T T Max gen: 112, Max flush: 199 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 111 111 111 196 196 196 F F F 2 040-059 112 112 112 199 199 199 F F F 3 060-079 108 108 108 190 190 190 F F F 4 080-099 100 100 100 172 172 172 F F F 5 100-119 110 110 110 193 193 193 F F F 6 120-139 113 113 113 200 200 200 F F F 7 140-159 111 111 111 196 196 196 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 110 110 110 193 193 193 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 110 110 110 193 193 193 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 113 113 113 200 200 200 F F F 14 280-299 113 113 113 200 200 200 F F F 15 300-319 113 113 113 200 200 200 F F F 16 320-339 110 110 110 194 194 194 F F F 17 340-359 110 110 110 193 193 193 F F F 18 360-379 108 108 108 190 190 190 F F F 19 380-399 113 113 113 200 200 200 F F F 20 400-419 113 113 113 200 200 200 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 112 112 112 199 199 199 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 112 112 112 199 199 199 F F F 27 540-559 113 113 113 200 200 200 F F F 28 560-579 113 113 113 200 200 200 F F F 29 580-599 113 113 113 200 200 200 F F F Max gen: 113, Max flush: 200 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.799528329Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.861486918Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.863441007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.863498856Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.863524291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.863548697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646} {"msg":"Crucible 85482006-60b7-4ac3-818b-9722b96d1095 has session id: bc428f26-2c19-40b6-9d52-a7d4403d9159","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.864293129Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Upstairs opts: Upstairs UUID: 85482006-60b7-4ac3-818b-9722b96d1095, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.864339251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Crucible stats registered with UUID: 85482006-60b7-4ac3-818b-9722b96d1095","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.864365706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:40.864474495Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865241034Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865275457Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646} {"msg":"85482006-60b7-4ac3-818b-9722b96d1095 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865341915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865376659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"0","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.86542264Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"0","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865450025Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"1","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.86547742Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"1","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865501296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"2","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865526421Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"2","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865549407Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865570203Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865594518Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"ds_connection connected from Ok(127.0.0.1:46860)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865712366Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"ds_connection connected from Ok(127.0.0.1:48747)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.8657485Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"ds_connection connected from Ok(127.0.0.1:47797)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.865774995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.866856815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"0","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.866896987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"2","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.86693666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"1","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.867552816Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"0","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.867630302Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"1","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.867683502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"2","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868261135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868369995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[0]R flush_numbers[0..12]: [181, 196, 199, 190, 172, 193, 200, 196, 193, 193, 190, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868439372Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[0]R generation[0..12]: [104, 111, 112, 108, 100, 110, 113, 111, 110, 110, 108, 110]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868460308Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868475535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[1]R flush_numbers[0..12]: [181, 196, 199, 190, 172, 193, 200, 196, 193, 193, 190, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868489593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[1]R generation[0..12]: [104, 111, 112, 108, 100, 110, 113, 111, 110, 110, 108, 110]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.86850372Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868518058Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[2]R flush_numbers[0..12]: [181, 196, 199, 190, 172, 193, 200, 196, 193, 193, 190, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868532335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[2]R generation[0..12]: [104, 111, 112, 108, 100, 110, 113, 111, 110, 110, 108, 110]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868545882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868567558Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868580946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868601652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:21:40.868626478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868656372Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"":"downstairs","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868672389Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868684567Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"0","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868697964Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"1","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868711732Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"client":"2","":"downstairs","session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868725089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868737737Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"85482006-60b7-4ac3-818b-9722b96d1095 is now active with session: bc428f26-2c19-40b6-9d52-a7d4403d9159","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868750655Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.868762902Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646,"session_id":"bc428f26-2c19-40b6-9d52-a7d4403d9159"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:40.86877494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 28672 data: 7 5 6 6 7 7 7 02/30 Write block 233 len 24576 data: 12 7 9 10 9 10 03/30 Flush 04/30 Write block 181 len 8192 data: 11 12 05/30 Read block 475 len 4096 06/30 Read block 480 len 16384 07/30 Read block 275 len 36864 08/30 Write block 51 len 28672 data: 9 8 7 7 6 6 6 09/30 Write block 580 len 12288 data: 11 12 11 10/30 Flush 11/30 Write block 584 len 40960 data: 8 6 6 6 6 7 6 4 4 5 12/30 Read block 146 len 36864 13/30 Read block 207 len 28672 14/30 Write block 138 len 20480 data: 9 11 8 6 8 15/30 Write block 42 len 36864 data: 7 5 8 9 9 7 8 12 9 16/30 Flush 17/30 Write block 537 len 28672 data: 7 8 9 9 9 13 12 18/30 Write block 32 len 24576 data: 9 8 6 5 5 7 19/30 Write block 369 len 40960 data: 12 11 12 14 11 12 14 16 13 13 20/30 Read block 174 len 12288 21/30 Write block 499 len 36864 data: 9 9 5 7 10 9 11 10 9 22/30 Write block 79 len 8192 data: 8 8 23/30 Read block 320 len 24576 24/30 Read block 581 len 24576 25/30 Read block 133 len 40960 26/30 Read block 252 len 36864 27/30 Write block 269 len 16384 data: 7 7 7 9 28/30 Read block 42 len 8192 29/30 Read block 231 len 32768 30/30 Read block 364 len 40960 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 13 13 14 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:40.894388622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 111 111 111 196 196 196 T T T 2 040-059 114 114 114 203 203 203 F F F 3 060-079 108 108 108 190 190 190 T T T 4 080-099 100 100 100 172 172 172 T T T 5 100-119 110 110 110 193 193 193 F F F 6 120-139 114 114 114 203 203 203 F F F 7 140-159 114 114 114 203 203 203 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 114 114 114 202 202 202 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 113 113 113 200 200 200 T T T 14 280-299 113 113 113 200 200 200 F F F 15 300-319 113 113 113 200 200 200 F F F 16 320-339 110 110 110 194 194 194 F F F 17 340-359 110 110 110 193 193 193 F F F 18 360-379 108 108 108 190 190 190 T T T 19 380-399 113 113 113 200 200 200 F F F 20 400-419 113 113 113 200 200 200 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 112 112 112 199 199 199 T T T 25 500-519 114 114 114 201 201 201 T T T 26 520-539 114 114 114 201 201 201 T T T 27 540-559 113 113 113 200 200 200 T T T 28 560-579 113 113 113 200 200 200 F F F 29 580-599 114 114 114 203 203 203 F F F Max gen: 114, Max flush: 203 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 115 115 115 204 204 204 F F F 4 080-099 115 115 115 204 204 204 F F F 5 100-119 110 110 110 193 193 193 F F F 6 120-139 114 114 114 203 203 203 F F F 7 140-159 114 114 114 203 203 203 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 114 114 114 202 202 202 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 113 113 113 200 200 200 F F F 15 300-319 113 113 113 200 200 200 F F F 16 320-339 110 110 110 194 194 194 F F F 17 340-359 110 110 110 193 193 193 F F F 18 360-379 115 115 115 204 204 204 F F F 19 380-399 113 113 113 200 200 200 F F F 20 400-419 113 113 113 200 200 200 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 115 115 115 204 204 204 F F F 25 500-519 115 115 115 204 204 204 F F F 26 520-539 115 115 115 204 204 204 F F F 27 540-559 115 115 115 204 204 204 F F F 28 560-579 113 113 113 200 200 200 F F F 29 580-599 114 114 114 203 203 203 F F F Max gen: 115, Max flush: 204 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.291693001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.349905753Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.351852823Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.351911122Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.351928159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.351943396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654} {"msg":"Crucible b837fc00-aa7c-4fc0-8be9-4c11dd9ad505 has session id: 93b6bade-9d57-42a8-9745-15971626c48e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.352750077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Upstairs opts: Upstairs UUID: b837fc00-aa7c-4fc0-8be9-4c11dd9ad505, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.352887892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Crucible stats registered with UUID: b837fc00-aa7c-4fc0-8be9-4c11dd9ad505","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.352919946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:45.352937602Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353682174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353718388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654} {"msg":"b837fc00-aa7c-4fc0-8be9-4c11dd9ad505 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353797083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.35381604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"0","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353848004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"0","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353862641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"1","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353877138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"1","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353890296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"2","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353904443Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"2","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353917331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353930988Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.353989338Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41512)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.354183052Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39668)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.354223844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51237)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.354253579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.35527304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"0","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.355301315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"2","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.356035219Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"2","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.35619419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"1","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.356221265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"0","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.356777622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.356929174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357101682Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"1","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[0]R flush_numbers[0..12]: [181, 204, 203, 204, 204, 193, 203, 203, 193, 202, 190, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357855553Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[0]R generation[0..12]: [104, 115, 114, 115, 115, 110, 114, 114, 110, 114, 108, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357881528Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357905463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[1]R flush_numbers[0..12]: [181, 204, 203, 204, 204, 193, 203, 203, 193, 202, 190, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.35792178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[1]R generation[0..12]: [104, 115, 114, 115, 115, 110, 114, 114, 110, 114, 108, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357936158Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357950165Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[2]R flush_numbers[0..12]: [181, 204, 203, 204, 204, 193, 203, 203, 193, 202, 190, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357964013Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[2]R generation[0..12]: [104, 115, 114, 115, 115, 110, 114, 114, 110, 114, 108, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.35797787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.357991837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358004415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358017863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.35803058Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358043398Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"":"downstairs","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358056265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358068173Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"Wait for a query_work_queue command to finish before sending IO 0","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358095128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"1","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358112295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"client":"2","":"downstairs","session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358125703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.35813794Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"b837fc00-aa7c-4fc0-8be9-4c11dd9ad505 is now active with session: 93b6bade-9d57-42a8-9745-15971626c48e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358150438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358162316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654,"session_id":"93b6bade-9d57-42a8-9745-15971626c48e"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:45.358174123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 397 len 32768 data: 5 7 10 12 13 11 11 11 02/30 Write block 293 len 24576 data: 9 7 9 8 10 6 03/30 Read block 288 len 12288 04/30 Read block 52 len 12288 05/30 Flush 06/30 Write block 319 len 12288 data: 12 13 11 07/30 Write block 127 len 8192 data: 10 8 08/30 Write block 545 len 36864 data: 13 11 9 10 6 7 6 7 8 09/30 Write block 153 len 24576 data: 9 9 9 7 10 9 10/30 Read block 541 len 20480 11/30 Read block 6 len 40960 12/30 Write block 94 len 36864 data: 10 8 6 7 8 8 8 7 6 13/30 Read block 474 len 28672 14/30 Read block 22 len 32768 15/30 Flush 16/30 Flush 17/30 Write block 108 len 36864 data: 10 9 10 7 7 10 12 13 12 18/30 Read block 440 len 4096 19/30 Write block 535 len 28672 data: 3 4 8 9 10 10 10 20/30 Read block 38 len 40960 21/30 Read block 314 len 24576 22/30 Read block 82 len 4096 23/30 Read block 95 len 12288 24/30 Write block 155 len 16384 data: 10 8 11 10 25/30 Read block 211 len 32768 26/30 Read block 23 len 12288 27/30 Write block 356 len 32768 data: 15 15 15 16 14 13 11 10 28/30 Read block 69 len 16384 29/30 Read block 204 len 12288 30/30 Read block 492 len 16384 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 13 14 14 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:45.381948486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 115 115 115 204 204 204 F F F 4 080-099 116 116 116 206 206 206 F F F 5 100-119 116 116 116 206 206 206 T T T 6 120-139 116 116 116 206 206 206 F F F 7 140-159 116 116 116 206 206 206 T T T 8 160-179 110 110 110 193 193 193 F F F 9 180-199 114 114 114 202 202 202 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 116 116 116 205 205 205 F F F 15 300-319 116 116 116 206 206 206 F F F 16 320-339 116 116 116 206 206 206 F F F 17 340-359 110 110 110 193 193 193 T T T 18 360-379 115 115 115 204 204 204 T T T 19 380-399 116 116 116 205 205 205 F F F 20 400-419 116 116 116 205 205 205 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 115 115 115 204 204 204 F F F 25 500-519 115 115 115 204 204 204 F F F 26 520-539 115 115 115 204 204 204 T T T 27 540-559 116 116 116 206 206 206 T T T 28 560-579 113 113 113 200 200 200 F F F 29 580-599 114 114 114 203 203 203 F F F Max gen: 116, Max flush: 206 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 115 115 115 204 204 204 F F F 4 080-099 116 116 116 206 206 206 F F F 5 100-119 117 117 117 207 207 207 F F F 6 120-139 116 116 116 206 206 206 F F F 7 140-159 117 117 117 207 207 207 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 114 114 114 202 202 202 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 114 114 114 201 201 201 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 116 116 116 205 205 205 F F F 15 300-319 116 116 116 206 206 206 F F F 16 320-339 116 116 116 206 206 206 F F F 17 340-359 117 117 117 207 207 207 F F F 18 360-379 117 117 117 207 207 207 F F F 19 380-399 116 116 116 205 205 205 F F F 20 400-419 116 116 116 205 205 205 F F F 21 420-439 108 108 108 190 190 190 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 115 115 115 204 204 204 F F F 25 500-519 115 115 115 204 204 204 F F F 26 520-539 117 117 117 207 207 207 F F F 27 540-559 117 117 117 207 207 207 F F F 28 560-579 113 113 113 200 200 200 F F F 29 580-599 114 114 114 203 203 203 F F F Max gen: 117, Max flush: 207 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.685493877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:49.728551372Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.728981713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.760140889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.760780151Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.760817953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.76083403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.760853407Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662} {"msg":"Crucible 739a00ef-2bb8-4a01-9165-e15b9fa63196 has session id: 24e0473a-0261-46fd-8a6d-88a35746dd41","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.761497108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Upstairs opts: Upstairs UUID: 739a00ef-2bb8-4a01-9165-e15b9fa63196, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.76153761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Crucible stats registered with UUID: 739a00ef-2bb8-4a01-9165-e15b9fa63196","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.761561906Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:49.761582282Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762001765Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762036538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662} {"msg":"739a00ef-2bb8-4a01-9165-e15b9fa63196 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762097477Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762128611Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"0","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762155676Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"0","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762170063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"1","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762184221Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"1","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762197288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"2","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762211176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"2","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762224103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762237921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762251718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"ds_connection connected from Ok(127.0.0.1:48040)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762370736Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"ds_connection connected from Ok(127.0.0.1:43704)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762408529Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"ds_connection connected from Ok(127.0.0.1:34298)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.762540915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.763174797Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"1","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.763204152Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"2","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.763328179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"0","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.763824737Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"2","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.7639176Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"1","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764054775Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"0","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764425076Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.76451209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[0]R flush_numbers[0..12]: [181, 204, 203, 204, 206, 207, 206, 207, 193, 202, 190, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764649215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[0]R generation[0..12]: [104, 115, 114, 115, 116, 117, 116, 117, 110, 114, 108, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764669891Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764684818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[1]R flush_numbers[0..12]: [181, 204, 203, 204, 206, 207, 206, 207, 193, 202, 190, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764698915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[1]R generation[0..12]: [104, 115, 114, 115, 116, 117, 116, 117, 110, 114, 108, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764712703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.76472639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[2]R flush_numbers[0..12]: [181, 204, 203, 204, 206, 207, 206, 207, 193, 202, 190, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764740178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[2]R generation[0..12]: [104, 115, 114, 115, 116, 117, 116, 117, 110, 114, 108, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764753425Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764771282Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.7647844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422Wait for a query_work_queue command to finish before sending IO ","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764809435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.76483387Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764859386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"":"downstairs","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764884821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764908207Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"0","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764934872Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"1","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764962397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"client":"2","":"downstairs","session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.764988502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.765012548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"739a00ef-2bb8-4a01-9165-e15b9fa63196 is now active with session: 24e0473a-0261-46fd-8a6d-88a35746dd41","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:21:49.765037403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.765060699Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662,"session_id":"24e0473a-0261-46fd-8a6d-88a35746dd41"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:49.765084524Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 358 len 16384 data: 16 17 15 14 02/30 Write block 561 len 20480 data: 13 11 12 13 14 03/30 Read block 454 len 16384 04/30 Write block 332 len 32768 data: 10 11 12 11 10 8 9 12 05/30 Read block 439 len 20480 06/30 Read block 351 len 4096 07/30 Read block 531 len 40960 08/30 Write block 219 len 36864 data: 7 6 4 6 6 5 4 5 6 09/30 Read block 166 len 28672 10/30 Read block 493 len 40960 11/30 Write block 588 len 8192 data: 7 8 12/30 Write block 14 len 36864 data: 13 14 11 11 11 10 10 13 11 13/30 Read block 308 len 28672 14/30 Read block 406 len 24576 15/30 Write block 201 len 36864 data: 7 10 11 10 9 11 11 7 8 16/30 Write block 383 len 12288 data: 9 10 8 17/30 Read block 475 len 16384 18/30 Read block 553 len 36864 19/30 Write block 546 len 28672 data: 12 10 11 7 8 7 8 20/30 Write block 206 len 36864 data: 12 12 8 9 8 9 8 7 6 21/30 Read block 212 len 4096 22/30 Write block 141 len 32768 data: 7 9 6 7 10 12 12 12 23/30 Read block 451 len 4096 24/30 Write block 144 len 40960 data: 8 11 13 13 13 10 9 10 10 10 25/30 Read block 492 len 20480 26/30 Write block 577 len 36864 data: 8 9 9 12 13 12 8 9 7 27/30 Read block 540 len 4096 28/30 Write block 421 len 24576 data: 8 6 9 13 8 9 29/30 Write block 33 len 16384 data: 9 7 6 6 30/30 Write block 344 len 28672 data: 15 12 12 11 10 10 8 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:49.786498293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 181 181 181 T T T 1 020-039 115 115 115 204 204 204 T T T 2 040-059 114 114 114 203 203 203 F F F 3 060-079 115 115 115 204 204 204 F F F 4 080-099 116 116 116 206 206 206 F F F 5 100-119 117 117 117 207 207 207 F F F 6 120-139 116 116 116 206 206 206 F F F 7 140-159 117 117 117 207 207 207 T T T 8 160-179 110 110 110 193 193 193 F F F 9 180-199 114 114 114 202 202 202 F F F 10 200-219 108 108 108 190 190 190 T T T 11 220-239 114 114 114 201 201 201 T T T 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 116 116 116 205 205 205 F F F 15 300-319 116 116 116 206 206 206 F F F 16 320-339 116 116 116 206 206 206 T T T 17 340-359 117 117 117 207 207 207 T T T 18 360-379 117 117 117 207 207 207 T T T 19 380-399 116 116 116 205 205 205 T T T 20 400-419 116 116 116 205 205 205 F F F 21 420-439 108 108 108 190 190 190 T T T 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 115 115 115 204 204 204 F F F 25 500-519 115 115 115 204 204 204 F F F 26 520-539 117 117 117 207 207 207 F F F 27 540-559 117 117 117 207 207 207 T T T 28 560-579 113 113 113 200 200 200 T T T 29 580-599 114 114 114 203 203 203 T T T Max gen: 117, Max flush: 207 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 208 208 208 F F F 1 020-039 118 118 118 208 208 208 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 115 115 115 204 204 204 F F F 4 080-099 116 116 116 206 206 206 F F F 5 100-119 117 117 117 207 207 207 F F F 6 120-139 116 116 116 206 206 206 F F F 7 140-159 118 118 118 208 208 208 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 114 114 114 202 202 202 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 118 118 118 208 208 208 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 116 116 116 205 205 205 F F F 15 300-319 116 116 116 206 206 206 F F F 16 320-339 118 118 118 208 208 208 F F F 17 340-359 118 118 118 208 208 208 F F F 18 360-379 118 118 118 208 208 208 F F F 19 380-399 118 118 118 208 208 208 F F F 20 400-419 116 116 116 205 205 205 F F F 21 420-439 118 118 118 208 208 208 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 F F F 24 480-499 115 115 115 204 204 204 F F F 25 500-519 115 115 115 204 204 204 F F F 26 520-539 117 117 117 207 207 207 F F F 27 540-559 118 118 118 208 208 208 F F F 28 560-579 118 118 118 208 208 208 F F F 29 580-599 118 118 118 208 208 208 F F F Max gen: 118, Max flush: 208 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.111235584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.158685127Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.159760718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.15980495Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.159827426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.159846292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670} {"msg":"Crucible 836e7a57-9f17-4996-a684-44e1c86c4de7 has session id: 984a3391-3d40-4612-a9f9-3813272d13fc","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.160519558Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Upstairs opts: Upstairs UUID: 836e7a57-9f17-4996-a684-44e1c86c4de7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.16056298Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Crucible stats registered with UUID: 836e7a57-9f17-4996-a684-44e1c86c4de7","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.160582766Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:54.160598463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161146222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161174297Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670} {"msg":"836e7a57-9f17-4996-a684-44e1c86c4de7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161241384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161259021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"0","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161285976Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"0","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161299623Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"1","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161313871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"1","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161331708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"2","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161354993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"2","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.16142488Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161461943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161491838Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35363)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161520742Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:58891)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161549337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:33914)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.161700179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.162327873Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"2","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.162356468Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"0","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.16304711Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"0","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.163086663Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"2","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.163693511Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.163720326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.164017231Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"1","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.164525447Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"1","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[0]R flush_numbers[0..12]: [208, 208, 203, 204, 206, 207, 206, 208, 193, 202, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165079334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[0]R generation[0..12]: [118, 118, 114, 115, 116, 117, 116, 118, 110, 114, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165110429Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165129045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[1]R flush_numbers[0..12]: [208, 208, 203, 204, 206, 207, 206, 208, 193, 202, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165146292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[1]R generation[0..12]: [118, 118, 114, 115, 116, 117, 116, 118, 110, 114, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165163129Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165180006Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[2]R flush_numbers[0..12]: [208, 208, 203, 204, 206, 207, 206, 208, 193, 202, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165196573Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[2]R generation[0..12]: [118, 118, 114, 115, 116, 117, 116, 118, 110, 114, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165220958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165237955Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Max found gen is 119","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165253702Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Generation requested: 120 >= found:119","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165270029Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165285656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165306622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"":"downstairs","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165323589Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165338376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"0","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165354653Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"1","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.16537051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"client":"2","":"downstairs","session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165386458Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165401215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"836e7a57-9f17-4996-a684-44e1c86c4de7 is now active with session: 984a3391-3d40-4612-a9f9-3813272d13fc","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165420591Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165435388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670,"session_id":"984a3391-3d40-4612-a9f9-3813272d13fc"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:21:54.165450395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 468 len 32768 data: 8 10 7 8 9 9 10 12 02/30 Write block 108 len 28672 data: 11 10 11 8 8 11 13 03/30 Write block 561 len 32768 data: 14 12 13 14 15 13 12 12 04/30 Read block 64 len 40960 05/30 Read block 207 len 8192 06/30 Write block 342 len 16384 data: 9 10 16 13 07/30 Read block 413 len 40960 08/30 Read block 289 len 4096 09/30 Read block 559 len 16384 10/30 Write block 529 len 36864 data: 7 6 7 6 5 4 4 5 9 11/30 Read block 569 len 16384 12/30 Write block 11 len 40960 data: 17 14 15 14 15 12 12 12 11 11 13/30 Read block 371 len 4096 14/30 Write block 71 len 4096 data: 20 15/30 Read block 550 len 40960 16/30 Write block 405 len 32768 data: 12 13 11 8 8 9 8 9 17/30 Read block 107 len 20480 18/30 Write block 391 len 36864 data: 11 11 10 9 7 6 6 8 11 19/30 Write block 478 len 28672 data: 11 10 11 10 10 10 13 20/30 Write block 365 len 20480 data: 11 11 14 12 13 21/30 Read block 7 len 4096 22/30 Read block 17 len 24576 23/30 Write block 569 len 28672 data: 8 8 6 7 10 11 10 24/30 Read block 578 len 36864 25/30 Write block 189 len 4096 data: 4 26/30 Write block 515 len 24576 data: 7 8 6 7 7 8 27/30 Write block 91 len 12288 data: 6 7 10 28/30 Write block 421 len 28672 data: 9 7 10 14 9 10 7 29/30 Read block 468 len 12288 30/30 Write block 187 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 3 Sent Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 1 6 Done 27 28 29 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:21:54.186101946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 208 208 208 T T T 1 020-039 118 118 118 208 208 208 T T T 2 040-059 114 114 114 203 203 203 F F F 3 060-079 115 115 115 204 204 204 T T T 4 080-099 116 116 116 206 206 206 T T T 5 100-119 117 117 117 207 207 207 T T T 6 120-139 116 116 116 206 206 206 F F F 7 140-159 118 118 118 208 208 208 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 114 114 114 202 202 202 T T T 10 200-219 118 118 118 208 208 208 F F F 11 220-239 118 118 118 208 208 208 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 116 116 116 205 205 205 F F F 15 300-319 116 116 116 206 206 206 F F F 16 320-339 118 118 118 208 208 208 F F F 17 340-359 118 118 118 208 208 208 T T T 18 360-379 118 118 118 208 208 208 T T T 19 380-399 118 118 118 208 208 208 T T T 20 400-419 116 116 116 205 205 205 T T T 21 420-439 118 118 118 208 208 208 T T T 22 440-459 104 104 104 182 182 182 F F F 23 460-479 112 112 112 199 199 199 T T T 24 480-499 115 115 115 204 204 204 T T T 25 500-519 115 115 115 204 204 204 T T T 26 520-539 117 117 117 207 207 207 T T T 27 540-559 118 118 118 208 208 208 F F F 28 560-579 118 118 118 208 208 208 T T T 29 580-599 118 118 118 208 208 208 F F F Max gen: 118, Max flush: 208 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 209 209 209 F F F 1 020-039 119 119 119 209 209 209 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 119 119 119 209 209 209 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 119 119 119 209 209 209 F F F 6 120-139 116 116 116 206 206 206 F F F 7 140-159 118 118 118 208 208 208 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 119 119 119 209 209 209 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 118 118 118 208 208 208 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 116 116 116 205 205 205 F F F 15 300-319 116 116 116 206 206 206 F F F 16 320-339 118 118 118 208 208 208 F F F 17 340-359 119 119 119 209 209 209 F F F 18 360-379 119 119 119 209 209 209 F F F 19 380-399 119 119 119 209 209 209 F F F 20 400-419 119 119 119 209 209 209 F F F 21 420-439 119 119 119 209 209 209 F F F 22 440-459 104 104 104 182 182 182 F F F 23 460-479 119 119 119 209 209 209 F F F 24 480-499 119 119 119 209 209 209 F F F 25 500-519 119 119 119 209 209 209 F F F 26 520-539 119 119 119 209 209 209 F F F 27 540-559 118 118 118 208 208 208 F F F 28 560-579 119 119 119 209 209 209 F F F 29 580-599 118 118 118 208 208 208 F F F Max gen: 119, Max flush: 209 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.510316554Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.556328313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.557396455Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.557441656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.557456334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.557470801Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678} {"msg":"Crucible 1c8df53f-d99e-4add-8ddb-39c40d3d8500 has session id: 7b9bea34-104c-4690-a3ca-d823e24d872a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558093926Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Upstairs opts: Upstairs UUID: 1c8df53f-d99e-4add-8ddb-39c40d3d8500, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558143547Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Crucible stats registered with UUID: 1c8df53f-d99e-4add-8ddb-39c40d3d8500","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558167962Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:08.558190728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558648393Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558683197Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678} {"msg":"1c8df53f-d99e-4add-8ddb-39c40d3d8500 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558745115Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558770301Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"0","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558798405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"0","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558812523Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.55882653Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558839588Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"2","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558853475Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"2","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558866593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.55888065Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.558952547Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43391)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.559027403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57542)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.559065046Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:08.55942228Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:08.559454444Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:08.559471591Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.55958324Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.559764917Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"2","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.559786713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"0","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.560437732Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"0","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.560477725Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"2","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.561122486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:08.561148681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.561542328Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37398)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.561688721Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.562843467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.563426919Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[0]R flush_numbers[0..12]: [209, 209, 203, 209, 209, 209, 206, 208, 193, 209, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564046035Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[0]R generation[0..12]: [119, 119, 114, 119, 119, 119, 116, 118, 110, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564069421Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564084268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[1]R flush_numbers[0..12]: [209, 209, 203, 209, 209, 209, 206, 208, 193, 209, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564098045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[1]R generation[0..12]: [119, 119, 114, 119, 119, 119, 116, 118, 110, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564111803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.5641257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[2]R flush_numbers[0..12]: [209, 209, 203, 209, 209, 209, 206, 208, 193, 209, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564139357Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[2]R generation[0..12]: [119, 119, 114, 119, 119, 119, 116, 118, 110, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564152545Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564165983Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.56417841Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Generation requested: 122 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564191118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564209364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564222602Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"":"downstairs","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:22:18.56423542Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564251327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"0","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564264294Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"1","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564276922Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"client":"2","":"downstairs","session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.56428975Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564301567Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"1c8df53f-d99e-4add-8ddb-39c40d3d8500 is now active with session: 7b9bea34-104c-4690-a3ca-d823e24d872a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564313995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564325643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678,"session_id":"7b9bea34-104c-4690-a3ca-d823e24d872a"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:18.564337521Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 511 len 36864 data: 8 7 8 7 8 9 7 8 8 02/30 Read block 285 len 8192 03/30 Read block 145 len 36864 04/30 Write block 18 len 24576 data: 13 12 12 14 12 13 05/30 Write block 299 len 20480 data: 5 5 6 4 3 06/30 Flush 07/30 Write block 121 len 40960 data: 12 10 9 10 9 9 11 9 7 5 08/30 Write block 371 len 4096 data: 13 09/30 Read block 587 len 4096 10/30 Write block 530 len 12288 data: 7 8 7 11/30 Read block 288 len 40960 12/30 Read block 552 len 32768 13/30 Read block 520 len 16384 14/30 Write block 449 len 4096 data: 8 15/30 Read block 109 len 12288 16/30 Read block 116 len 16384 17/30 Read block 579 len 8192 18/30 Read block 350 len 12288 19/30 Read block 371 len 28672 20/30 Write block 399 len 8192 data: 12 13 21/30 Read block 351 len 28672 22/30 Read block 11 len 4096 23/30 Read block 248 len 36864 24/30 Read block 541 len 40960 25/30 Read block 2 len 16384 26/30 Read block 1 len 8192 27/30 Read block 421 len 40960 28/30 Write block 75 len 4096 data: 13 29/30 Read block 490 len 4096 30/30 Read block 513 len 24576 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 10 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 23 24 23 70 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:18.585404614Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 122 122 122 210 210 210 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 119 119 119 209 209 209 T T T 4 080-099 119 119 119 209 209 209 F F F 5 100-119 119 119 119 209 209 209 F F F 6 120-139 116 116 116 206 206 206 T T T 7 140-159 118 118 118 208 208 208 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 119 119 119 209 209 209 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 118 118 118 208 208 208 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 122 122 122 210 210 210 F F F 15 300-319 122 122 122 210 210 210 F F F 16 320-339 118 118 118 208 208 208 F F F 17 340-359 119 119 119 209 209 209 F F F 18 360-379 119 119 119 209 209 209 T T T 19 380-399 119 119 119 209 209 209 T T T 20 400-419 119 119 119 209 209 209 T T T 21 420-439 119 119 119 209 209 209 F F F 22 440-459 104 104 104 182 182 182 T T T 23 460-479 119 119 119 209 209 209 F F F 24 480-499 119 119 119 209 209 209 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 119 119 119 209 209 209 T T T 27 540-559 118 118 118 208 208 208 F F F 28 560-579 119 119 119 209 209 209 F F F 29 580-599 118 118 118 208 208 208 F F F Max gen: 122, Max flush: 210 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 122 122 122 210 210 210 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 123 123 123 211 211 211 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 119 119 119 209 209 209 F F F 6 120-139 123 123 123 211 211 211 F F F 7 140-159 118 118 118 208 208 208 F F F 8 160-179 110 110 110 193 193 193 F F F 9 180-199 119 119 119 209 209 209 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 118 118 118 208 208 208 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 115 115 115 204 204 204 F F F 14 280-299 122 122 122 210 210 210 F F F 15 300-319 122 122 122 210 210 210 F F F 16 320-339 118 118 118 208 208 208 F F F 17 340-359 119 119 119 209 209 209 F F F 18 360-379 123 123 123 211 211 211 F F F 19 380-399 123 123 123 211 211 211 F F F 20 400-419 123 123 123 211 211 211 F F F 21 420-439 119 119 119 209 209 209 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 119 119 119 209 209 209 F F F 24 480-499 119 119 119 209 209 209 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 123 123 123 211 211 211 F F F 27 540-559 118 118 118 208 208 208 F F F 28 560-579 119 119 119 209 209 209 F F F 29 580-599 118 118 118 208 208 208 F F F Max gen: 123, Max flush: 211 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.849807115Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.895612172Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.896537781Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.896591411Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.896610647Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.896628534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686} {"msg":"Crucible 9ed8b67f-5650-44ea-af55-34f715b9f33b has session id: 45ec6b8d-5160-4f97-a031-ff4992fdd1f7","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.897151997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Upstairs opts: Upstairs UUID: 9ed8b67f-5650-44ea-af55-34f715b9f33b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.897280923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Crucible stats registered with UUID: 9ed8b67f-5650-44ea-af55-34f715b9f33b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.897307039Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:22.897320716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.897909557Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.897943371Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686} {"msg":"9ed8b67f-5650-44ea-af55-34f715b9f33b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898025686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898043902Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"0","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898070588Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"0","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898085075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"1","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898099392Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"1","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898114329Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"2","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898139825Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"2","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.89816397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898189186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898215281Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51082)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898285118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43826)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.89832918Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53774)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898356295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.898997616Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"0","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.899025521Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"2","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.899664413Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"0","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.899701986Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"2","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.900287817Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.900313973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.900564466Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"1","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901144869Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"1","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[0]R flush_numbers[0..12]: [210, 210, 203, 211, 209, 209, 211, 208, 193, 209, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.90167659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[0]R generation[0..12]: [122, 122, 114, 123, 119, 119, 123, 118, 110, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901696107Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901710264Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[1]R flush_numbers[0..12]: [210, 210, 203, 211, 209, 209, 211, 208, 193, 209, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901723802Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[1]R generation[0..12]: [122, 122, 114, 123, 119, 119, 123, 118, 110, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901741198Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901755316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[2]R flush_numbers[0..12]: [210, 210, 203, 211, 209, 209, 211, 208, 193, 209, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901768713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[2]R generation[0..12]: [122, 122, 114, 123, 119, 119, 123, 118, 110, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901781921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901795388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901807966Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-07T18:22:22.901827793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.90184134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901853718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"":"downstairs","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901866465Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901878253Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"0","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901891741Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"1","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901904478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"client":"2","":"downstairs","session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901917196Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901928874Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"9ed8b67f-5650-44ea-af55-34f715b9f33b is now active with session: 45ec6b8d-5160-4f97-a031-ff4992fdd1f7","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901943571Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901955569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686,"session_id":"45ec6b8d-5160-4f97-a031-ff4992fdd1f7"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:22.901967427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 474 len 28672 data: 11 13 11 10 12 11 12 02/30 Write block 149 len 24576 data: 11 10 11 11 11 10 03/30 Flush 04/30 Write block 479 len 36864 data: 12 13 11 11 11 14 12 13 12 05/30 Read block 441 len 24576 06/30 Write block 37 len 8192 data: 8 8 07/30 Write block 478 len 40960 data: 13 13 14 12 12 12 15 13 14 13 08/30 Read block 530 len 8192 09/30 Read block 0 len 36864 10/30 Flush 11/30 Write block 313 len 28672 data: 13 11 10 12 10 12 13 12/30 Read block 161 len 28672 13/30 Read block 419 len 8192 14/30 Read block 284 len 12288 15/30 Write block 73 len 16384 data: 20 19 14 12 16/30 Read block 25 len 24576 17/30 Read block 29 len 36864 18/30 Write block 576 len 20480 data: 8 9 10 10 13 19/30 Read block 179 len 40960 20/30 Write block 28 len 4096 data: 6 21/30 Write block 272 len 24576 data: 10 7 6 6 6 8 22/30 Write block 154 len 24576 data: 11 11 9 12 11 6 23/30 Read block 572 len 24576 24/30 Flush 25/30 Write block 289 len 4096 data: 8 26/30 Write block 308 len 32768 data: 11 9 11 11 12 14 12 11 27/30 Write block 179 len 12288 data: 14 14 12 28/30 Read block 118 len 36864 29/30 Read block 162 len 40960 30/30 Write block 579 len 16384 data: 11 14 14 13 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:22.922517405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 124 124 124 214 214 214 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 119 119 119 209 209 209 F F F 6 120-139 123 123 123 211 211 211 F F F 7 140-159 124 124 124 214 214 214 F F F 8 160-179 110 110 110 193 193 193 T T T 9 180-199 119 119 119 209 209 209 T T T 10 200-219 118 118 118 208 208 208 F F F 11 220-239 118 118 118 208 208 208 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 124 124 124 214 214 214 F F F 14 280-299 122 122 122 210 210 210 T T T 15 300-319 124 124 124 214 214 214 T T T 16 320-339 118 118 118 208 208 208 F F F 17 340-359 119 119 119 209 209 209 F F F 18 360-379 123 123 123 211 211 211 F F F 19 380-399 123 123 123 211 211 211 F F F 20 400-419 123 123 123 211 211 211 F F F 21 420-439 119 119 119 209 209 209 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 124 124 124 213 213 213 F F F 24 480-499 124 124 124 213 213 213 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 123 123 123 211 211 211 F F F 27 540-559 118 118 118 208 208 208 F F F 28 560-579 124 124 124 214 214 214 T T T 29 580-599 124 124 124 214 214 214 T T T Max gen: 124, Max flush: 214 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 124 124 124 214 214 214 F F F 2 040-059 114 114 114 203 203 203 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 119 119 119 209 209 209 F F F 6 120-139 123 123 123 211 211 211 F F F 7 140-159 124 124 124 214 214 214 F F F 8 160-179 125 125 125 215 215 215 F F F 9 180-199 125 125 125 215 215 215 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 118 118 118 208 208 208 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 124 124 124 214 214 214 F F F 14 280-299 125 125 125 215 215 215 F F F 15 300-319 125 125 125 215 215 215 F F F 16 320-339 118 118 118 208 208 208 F F F 17 340-359 119 119 119 209 209 209 F F F 18 360-379 123 123 123 211 211 211 F F F 19 380-399 123 123 123 211 211 211 F F F 20 400-419 123 123 123 211 211 211 F F F 21 420-439 119 119 119 209 209 209 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 124 124 124 213 213 213 F F F 24 480-499 124 124 124 213 213 213 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 123 123 123 211 211 211 F F F 27 540-559 118 118 118 208 208 208 F F F 28 560-579 125 125 125 215 215 215 F F F 29 580-599 125 125 125 215 215 215 F F F Max gen: 125, Max flush: 215 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.193378123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:27.239442493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.239946799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.272276629Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.273195649Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.273232812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.273246909Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.273261677Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694} {"msg":"Crucible 45a45ff4-a101-418c-bdcb-e23250b52d4e has session id: fe2c0a70-0958-45ef-91ff-5a2883337ab5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.273817334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Upstairs opts: Upstairs UUID: 45a45ff4-a101-418c-bdcb-e23250b52d4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.273857546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Crucible stats registered with UUID: 45a45ff4-a101-418c-bdcb-e23250b52d4e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.273882911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:27.273898189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274331748Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274357004Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694} {"msg":"45a45ff4-a101-418c-bdcb-e23250b52d4e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274425021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274443518Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"0","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274469803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"0","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274483191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"1","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274501577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"1","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274516404Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"2","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274530432Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"2","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274543169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274559277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274582152Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40980)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274609277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38723)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274843734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38033)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.274882227Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.275377105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"1","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.27540644Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"2","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.275684148Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"0","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276138194Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"2","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276176877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"1","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276282778Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"0","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276725576Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276813089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[0]R flush_numbers[0..12]: [210, 214, 203, 214, 209, 209, 211, 214, 215, 215, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276930418Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[0]R generation[0..12]: [122, 124, 114, 124, 119, 119, 123, 124, 125, 125, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276949894Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276964022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[1]R flush_numbers[0..12]: [210, 214, 203, 214, 209, 209, 211, 214, 215, 215, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276977629Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[1]R generation[0..12]: [122, 124, 114, 124, 119, 119, 123, 124, 125, 125, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.276991376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.27702421Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[2]R flush_numbers[0..12]: [210, 214, 203, 214, 209, 209, 211, 214, 215, 215, 208, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277038418Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[2]R generation[0..12]: [122, 124, 114, 124, 119, 119, 123, 124, 125, 125, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277051985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277065433Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277091408Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277116564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277146048Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277171363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"":"downstairs","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277196869Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277220384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"0","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.27724598Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"1","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-07T18:22:27.277272285Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"client":"2","":"downstairs","session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277304499Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277326525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"45a45ff4-a101-418c-bdcb-e23250b52d4e is now active with session: fe2c0a70-0958-45ef-91ff-5a2883337ab5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277340022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.27735201Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694,"session_id":"fe2c0a70-0958-45ef-91ff-5a2883337ab5"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:27.277364118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 48 len 28672 data: 9 13 10 10 9 8 8 02/30 Write block 575 len 32768 data: 11 9 10 11 12 15 15 14 03/30 Write block 142 len 36864 data: 10 7 9 12 14 14 14 12 11 04/30 Write block 233 len 8192 data: 13 8 05/30 Write block 102 len 32768 data: 7 6 5 6 6 7 12 11 06/30 Read block 10 len 16384 07/30 Flush 08/30 Write block 319 len 16384 data: 14 14 12 10 09/30 Read block 306 len 28672 10/30 Read block 228 len 4096 11/30 Read block 525 len 20480 12/30 Read block 99 len 16384 13/30 Read block 121 len 16384 14/30 Read block 557 len 8192 15/30 Flush 16/30 Write block 536 len 20480 data: 6 10 10 11 11 17/30 Read block 504 len 16384 18/30 Read block 451 len 8192 19/30 Read block 69 len 28672 20/30 Flush 21/30 Write block 311 len 32768 data: 12 13 15 13 12 13 11 13 22/30 Read block 440 len 12288 23/30 Read block 209 len 32768 24/30 Write block 338 len 12288 data: 10 13 10 25/30 Read block 391 len 40960 26/30 Read block 486 len 36864 27/30 Write block 332 len 8192 data: 11 12 28/30 Write block 116 len 36864 data: 13 10 10 12 16 13 11 10 11 29/30 Read block 417 len 20480 30/30 Read block 179 len 24576 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:27.297481456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 124 124 124 214 214 214 F F F 2 040-059 126 126 126 216 216 216 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 126 126 126 216 216 216 T T T 6 120-139 123 123 123 211 211 211 T T T 7 140-159 126 126 126 216 216 216 F F F 8 160-179 125 125 125 215 215 215 F F F 9 180-199 125 125 125 215 215 215 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 126 126 126 216 216 216 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 124 124 124 214 214 214 F F F 14 280-299 125 125 125 215 215 215 F F F 15 300-319 126 126 126 217 217 217 T T T 16 320-339 126 126 126 217 217 217 T T T 17 340-359 119 119 119 209 209 209 T T T 18 360-379 123 123 123 211 211 211 F F F 19 380-399 123 123 123 211 211 211 F F F 20 400-419 123 123 123 211 211 211 F F F 21 420-439 119 119 119 209 209 209 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 124 124 124 213 213 213 F F F 24 480-499 124 124 124 213 213 213 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 126 126 126 218 218 218 F F F 27 540-559 126 126 126 218 218 218 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 126 126 126 216 216 216 F F F Max gen: 126, Max flush: 218 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 124 124 124 214 214 214 F F F 2 040-059 126 126 126 216 216 216 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 127 127 127 219 219 219 F F F 6 120-139 127 127 127 219 219 219 F F F 7 140-159 126 126 126 216 216 216 F F F 8 160-179 125 125 125 215 215 215 F F F 9 180-199 125 125 125 215 215 215 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 126 126 126 216 216 216 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 124 124 124 214 214 214 F F F 14 280-299 125 125 125 215 215 215 F F F 15 300-319 127 127 127 219 219 219 F F F 16 320-339 127 127 127 219 219 219 F F F 17 340-359 127 127 127 219 219 219 F F F 18 360-379 123 123 123 211 211 211 F F F 19 380-399 123 123 123 211 211 211 F F F 20 400-419 123 123 123 211 211 211 F F F 21 420-439 119 119 119 209 209 209 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 124 124 124 213 213 213 F F F 24 480-499 124 124 124 213 213 213 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 126 126 126 218 218 218 F F F 27 540-559 126 126 126 218 218 218 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 126 126 126 216 216 216 F F F Max gen: 127, Max flush: 219 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.585793986Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.632405763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.633529606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.633583376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.633602822Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.633620559Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702} {"msg":"Crucible 9cc18e6c-dc12-41f9-b141-c29b2c2183c1 has session id: bb56ab0e-ff97-40b8-bee3-f67ffc832c43","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634238124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Upstairs opts: Upstairs UUID: 9cc18e6c-dc12-41f9-b141-c29b2c2183c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634278807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Crucible stats registered with UUID: 9cc18e6c-dc12-41f9-b141-c29b2c2183c1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634295064Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:31.634308742Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634770586Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634808229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702} {"msg":"9cc18e6c-dc12-41f9-b141-c29b2c2183c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634882395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.63490989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"0","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634938295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"0","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634952323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"1","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.63496649Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"1","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634979458Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.634993395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.635006682Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.63502043Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.635035317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:39730)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.635194308Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:35717)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.635242299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:31.635270504Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:31.635304547Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:31.635448931Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.635473536Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.636009087Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"0","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.636036342Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"1","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.636646639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"1","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.636675544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"0","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.637264405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:31.6372896Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.636396695Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:39579)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.636557945Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.637765262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.638338586Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[0]R flush_numbers[0..12]: [210, 214, 216, 214, 209, 219, 219, 216, 215, 215, 208, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.638969099Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[0]R generation[0..12]: [122, 124, 126, 124, 119, 127, 127, 126, 125, 125, 118, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639001903Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.63901805Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[1]R flush_numbers[0..12]: [210, 214, 216, 214, 209, 219, 219, 216, 215, 215, 208, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639032857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[1]R generation[0..12]: [122, 124, 126, 124, 119, 127, 127, 126, 125, 125, 118, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639046695Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639060572Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[2]R flush_numbers[0..12]: [210, 214, 216, 214, 209, 219, 219, 216, 215, 215, 208, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.63907413Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[2]R generation[0..12]: [122, 124, 126, 124, 119, 127, 127, 126, 125, 125, 118, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639088497Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639102475Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639132799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639147336Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639160074Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639172651Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"":"downstairs","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639185229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639197867Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"0","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639210915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"1","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639227041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"client":"2","":"downstairs","session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639240109Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:22:41.639251827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"9cc18e6c-dc12-41f9-b141-c29b2c2183c1 is now active with session: bb56ab0e-ff97-40b8-bee3-f67ffc832c43","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639272973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639284621Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702,"session_id":"bb56ab0e-ff97-40b8-bee3-f67ffc832c43"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:41.639296469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 407 len 16384 data: 12 9 9 10 02/30 Write block 297 len 36864 data: 11 7 6 6 7 5 4 4 6 03/30 Read block 357 len 20480 04/30 Read block 393 len 12288 05/30 Write block 353 len 32768 data: 9 9 11 16 16 17 18 16 06/30 Read block 500 len 4096 07/30 Read block 151 len 8192 08/30 Read block 427 len 40960 09/30 Read block 454 len 16384 10/30 Read block 509 len 36864 11/30 Read block 346 len 40960 12/30 Write block 388 len 8192 data: 8 10 13/30 Write block 167 len 28672 data: 7 7 9 10 10 12 11 14/30 Read block 250 len 8192 15/30 Read block 423 len 40960 16/30 Read block 464 len 28672 17/30 Write block 130 len 4096 data: 6 18/30 Read block 393 len 32768 19/30 Read block 314 len 16384 20/30 Write block 422 len 8192 data: 8 11 21/30 Write block 490 len 32768 data: 10 10 11 10 10 11 10 9 22/30 Read block 157 len 24576 23/30 Write block 464 len 28672 data: 10 9 7 8 9 11 8 24/30 Read block 423 len 36864 25/30 Write block 22 len 24576 data: 13 14 11 12 8 7 26/30 Read block 310 len 20480 27/30 Read block 274 len 40960 28/30 Read block 381 len 12288 29/30 Read block 175 len 8192 30/30 Read block 506 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:41.662388277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 124 124 124 214 214 214 T T T 2 040-059 126 126 126 216 216 216 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 127 127 127 219 219 219 F F F 6 120-139 127 127 127 219 219 219 T T T 7 140-159 126 126 126 216 216 216 F F F 8 160-179 125 125 125 215 215 215 T T T 9 180-199 125 125 125 215 215 215 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 126 126 126 216 216 216 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 124 124 124 214 214 214 F F F 14 280-299 125 125 125 215 215 215 T T T 15 300-319 127 127 127 219 219 219 T T T 16 320-339 127 127 127 219 219 219 F F F 17 340-359 127 127 127 219 219 219 T T T 18 360-379 123 123 123 211 211 211 T T T 19 380-399 123 123 123 211 211 211 T T T 20 400-419 123 123 123 211 211 211 T T T 21 420-439 119 119 119 209 209 209 T T T 22 440-459 123 123 123 211 211 211 F F F 23 460-479 124 124 124 213 213 213 T T T 24 480-499 124 124 124 213 213 213 T T T 25 500-519 122 122 122 210 210 210 F F F 26 520-539 126 126 126 218 218 218 F F F 27 540-559 126 126 126 218 218 218 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 126 126 126 216 216 216 F F F Max gen: 127, Max flush: 219 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 128 128 128 220 220 220 F F F 2 040-059 126 126 126 216 216 216 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 119 119 119 209 209 209 F F F 5 100-119 127 127 127 219 219 219 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 126 126 126 216 216 216 F F F 8 160-179 128 128 128 220 220 220 F F F 9 180-199 125 125 125 215 215 215 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 126 126 126 216 216 216 F F F 12 240-259 100 100 100 172 172 172 F F F 13 260-279 124 124 124 214 214 214 F F F 14 280-299 128 128 128 220 220 220 F F F 15 300-319 128 128 128 220 220 220 F F F 16 320-339 127 127 127 219 219 219 F F F 17 340-359 128 128 128 220 220 220 F F F 18 360-379 128 128 128 220 220 220 F F F 19 380-399 128 128 128 220 220 220 F F F 20 400-419 128 128 128 220 220 220 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 128 128 128 220 220 220 F F F 24 480-499 128 128 128 220 220 220 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 126 126 126 218 218 218 F F F 27 540-559 126 126 126 218 218 218 F F F 28 560-579 126 126 126 216 216 216 F F F 29 580-599 126 126 126 216 216 216 F F F Max gen: 128, Max flush: 220 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:55.956262168Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.003426413Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.004447284Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.004492776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.004507503Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.004522091Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710} {"msg":"Crucible 10929624-7a23-4f4e-aac6-b83f3cfe6fe6 has session id: ec4ce8eb-7830-4930-8f9e-3ce45e304780","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005136727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Upstairs opts: Upstairs UUID: 10929624-7a23-4f4e-aac6-b83f3cfe6fe6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005178279Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Crucible stats registered with UUID: 10929624-7a23-4f4e-aac6-b83f3cfe6fe6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005201955Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:56.005222821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005790426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005826239Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710} {"msg":"10929624-7a23-4f4e-aac6-b83f3cfe6fe6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005907284Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005929Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"0","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005956335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"0","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005970412Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"1","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.00598453Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"1","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.005997638Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"2","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006011305Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"2","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006024123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006045199Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006071234Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"ds_connection connected from Ok(127.0.0.1:60718)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006095509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"ds_connection connected from Ok(127.0.0.1:60169)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006156338Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"ds_connection connected from Ok(127.0.0.1:54405)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006347493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.006877015Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"0","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.00690521Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"2","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.007061981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"1","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.00760535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"2","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.007680786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"1","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.00771233Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"0","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008352232Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008451604Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 216, 214, 209, 219, 220, 216, 220, 215, 208, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008516952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[0]R generation[0..12]: [122, 128, 126, 124, 119, 127, 128, 126, 128, 125, 118, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008538388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008553815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 216, 214, 209, 219, 220, 216, 220, 215, 208, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008567692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[1]R generation[0..12]: [122, 128, 126, 124, 119, 127, 128, 126, 128, 125, 118, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008581349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008595097Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 216, 214, 209, 219, 220, 216, 220, 215, 208, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008613594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[2]R generation[0..12]: [122, 128, 126, 124, 119, 127, 128, 126, 128, 125, 118, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008627481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008641189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Max found gen is 129","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008653786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Generation requested: 130 >= found:129","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008666714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008679382Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008691909Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"":"downstairs","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008712045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008723673Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"0","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008736651Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"1","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008749538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"client":"2","":"downstairs","session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008762256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008774124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"10929624-7a23-4f4e-aac6-b83f3cfe6fe6 is now active with session: ec4ce8eb-7830-4930-8f9e-3ce45e304780","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008786542Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008798169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710,"session_id":"ec4ce8eb-7830-4930-8f9e-3ce45e304780"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:22:56.008812417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 16384 data: 13 14 16 14 02/30 Read block 28 len 4096 03/30 Read block 331 len 28672 04/30 Write block 546 len 12288 data: 13 11 12 05/30 Write block 366 len 8192 data: 12 15 06/30 Read block 170 len 36864 07/30 Write block 80 len 36864 data: 9 9 11 9 11 12 9 11 9 08/30 Write block 36 len 40960 data: 7 9 9 7 10 8 8 6 9 10 09/30 Flush 10/30 Write block 52 len 32768 data: 10 9 9 7 7 7 5 5 11/30 Read block 471 len 4096 12/30 Read block 522 len 24576 13/30 Read block 53 len 4096 14/30 Flush 15/30 Write block 239 len 20480 data: 10 8 7 6 7 16/30 Read block 180 len 24576 17/30 Write block 471 len 24576 data: 9 10 10 12 14 12 18/30 Write block 406 len 24576 data: 14 13 10 10 11 9 19/30 Read block 453 len 36864 20/30 Read block 566 len 16384 21/30 Read block 147 len 8192 22/30 Write block 562 len 8192 data: 13 14 23/30 Write block 522 len 12288 data: 8 8 11 24/30 Read block 191 len 12288 25/30 Read block 399 len 32768 26/30 Write block 323 len 8192 data: 10 10 27/30 Write block 313 len 32768 data: 17 15 13 14 12 14 15 15 28/30 Read block 290 len 28672 29/30 Read block 106 len 24576 30/30 Read block 301 len 28672 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 16 15 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:22:56.029527425Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 130 130 130 221 221 221 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 130 130 130 221 221 221 F F F 5 100-119 127 127 127 219 219 219 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 126 126 126 216 216 216 F F F 8 160-179 128 128 128 220 220 220 F F F 9 180-199 125 125 125 215 215 215 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 126 126 126 216 216 216 T T T 12 240-259 100 100 100 172 172 172 T T T 13 260-279 124 124 124 214 214 214 F F F 14 280-299 128 128 128 220 220 220 F F F 15 300-319 130 130 130 221 221 221 T T T 16 320-339 127 127 127 219 219 219 T T T 17 340-359 128 128 128 220 220 220 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 128 128 128 220 220 220 F F F 20 400-419 128 128 128 220 220 220 T T T 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 128 128 128 220 220 220 T T T 24 480-499 128 128 128 220 220 220 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 126 126 126 218 218 218 T T T 27 540-559 130 130 130 221 221 221 F F F 28 560-579 126 126 126 216 216 216 T T T 29 580-599 126 126 126 216 216 216 F F F Max gen: 130, Max flush: 222 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 F F F 1 020-039 130 130 130 221 221 221 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 130 130 130 221 221 221 F F F 5 100-119 127 127 127 219 219 219 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 126 126 126 216 216 216 F F F 8 160-179 128 128 128 220 220 220 F F F 9 180-199 125 125 125 215 215 215 F F F 10 200-219 118 118 118 208 208 208 F F F 11 220-239 131 131 131 223 223 223 F F F 12 240-259 131 131 131 223 223 223 F F F 13 260-279 124 124 124 214 214 214 F F F 14 280-299 128 128 128 220 220 220 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 131 131 131 223 223 223 F F F 17 340-359 128 128 128 220 220 220 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 128 128 128 220 220 220 F F F 20 400-419 131 131 131 223 223 223 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 131 131 131 223 223 223 F F F 24 480-499 128 128 128 220 220 220 F F F 25 500-519 122 122 122 210 210 210 F F F 26 520-539 131 131 131 223 223 223 F F F 27 540-559 130 130 130 221 221 221 F F F 28 560-579 131 131 131 223 223 223 F F F 29 580-599 126 126 126 216 216 216 F F F Max gen: 131, Max flush: 223 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.298203697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.344792469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.345621295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.345666537Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.345686883Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.345701611Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718} {"msg":"Crucible c0f06ab7-c8ba-419a-897c-e195152a632b has session id: a66a9c73-6895-48c8-a7fd-c7119a32a38f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.346306939Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Upstairs opts: Upstairs UUID: c0f06ab7-c8ba-419a-897c-e195152a632b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.346344202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Crucible stats registered with UUID: c0f06ab7-c8ba-419a-897c-e195152a632b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.346370007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:10.346390593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.346885412Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.346911887Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718} {"msg":"c0f06ab7-c8ba-419a-897c-e195152a632b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.346975445Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.34700285Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"0","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347030275Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"0","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.34705672Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"1","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347080616Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"1","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347096282Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"2","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347115229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"2","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347128556Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347148353Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347172588Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50398)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347319811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50636)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347364703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40595)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.347396957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.348156346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"0","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.34819148Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"1","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.348799178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"1","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.348960098Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"0","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.349027415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"2","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.349453326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.349578573Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.349859302Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"2","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[0]R flush_numbers[0..12]: [210, 221, 222, 214, 221, 219, 220, 216, 220, 215, 208, 223]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.35040449Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[0]R generation[0..12]: [122, 130, 130, 124, 130, 127, 128, 126, 128, 125, 118, 131]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350432496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350447673Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[1]R flush_numbers[0..12]: [210, 221, 222, 214, 221, 219, 220, 216, 220, 215, 208, 223]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.35046132Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[1]R generation[0..12]: [122, 130, 130, 124, 130, 127, 128, 126, 128, 125, 118, 131]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350475148Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350488945Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[2]R flush_numbers[0..12]: [210, 221, 222, 214, 221, 219, 220, 216, 220, 215, 208, 223]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350502423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[2]R generation[0..12]: [122, 130, 130, 124, 130, 127, 128, 126, 128, 125, 118, 131]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.35051576Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350529278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350548064Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350561112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350573689Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350586197Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"":"downstairs","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350599185Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350615791Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"0","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350629549Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"1","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350642606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"client":"2","":"downstairs","session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350655544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350667402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"c0f06ab7-c8ba-419a-897c-e195152a632b is now active with session: a66a9c73-6895-48c8-a7fd-c7119a32a38f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.35067972Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350726271Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JNRWVR4SNXKEADEZ76B93422","pid":1718,"session_id":"a66a9c73-6895-48c8-a7fd-c7119a32a38f"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:10.350790399Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 250 len 4096 data: 9 02/30 Read block 181 len 20480 03/30 Flush 04/30 Write block 276 len 24576 data: 7 9 8 9 10 7 05/30 Read block 330 len 20480 06/30 Write block 29 len 20480 data: 9 9 8 10 10 07/30 Write block 146 len 8192 data: 15 15 08/30 Write block 230 len 40960 data: 10 11 12 14 9 10 11 10 11 11 09/30 Read block 403 len 4096 10/30 Read block 335 len 16384 11/30 Write block 196 len 40960 data: 10 9 8 7 6 8 11 12 11 10 12/30 Read block 474 len 24576 13/30 Read block 62 len 32768 14/30 Write block 87 len 24576 data: 12 10 10 8 7 8 15/30 Read block 40 len 24576 16/30 Write block 276 len 20480 data: 8 10 9 10 11 17/30 Write block 547 len 8192 data: 12 13 18/30 Write block 550 len 24576 data: 9 8 9 9 7 7 19/30 Flush 20/30 Write block 577 len 36864 data: 11 12 13 16 16 15 9 10 8 21/30 Read block 359 len 24576 22/30 Write block 65 len 8192 data: 11 9 23/30 Read block 369 len 32768 24/30 Read block 254 len 20480 25/30 Write block 496 len 40960 data: 11 10 9 10 10 6 8 11 10 12 26/30 Write block 351 len 20480 data: 7 7 10 10 12 27/30 Write block 8 len 24576 data: 13 18 17 18 15 16 28/30 Read block 468 len 12288 29/30 Write block 246 len 20480 data: 8 9 10 10 10 30/30 Read block 218 len 36864 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:10.37208347Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 210 210 210 T T T 1 020-039 132 132 132 225 225 225 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 124 124 124 214 214 214 T T T 4 080-099 132 132 132 225 225 225 F F F 5 100-119 127 127 127 219 219 219 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 132 132 132 225 225 225 F F F 8 160-179 128 128 128 220 220 220 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 132 132 132 225 225 225 F F F 11 220-239 132 132 132 225 225 225 F F F 12 240-259 132 132 132 224 224 224 T T T 13 260-279 132 132 132 225 225 225 F F F 14 280-299 132 132 132 225 225 225 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 131 131 131 223 223 223 F F F 17 340-359 128 128 128 220 220 220 T T T 18 360-379 130 130 130 221 221 221 F F F 19 380-399 128 128 128 220 220 220 F F F 20 400-419 131 131 131 223 223 223 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 131 131 131 223 223 223 F F F 24 480-499 128 128 128 220 220 220 T T T 25 500-519 122 122 122 210 210 210 T T T 26 520-539 131 131 131 223 223 223 F F F 27 540-559 132 132 132 225 225 225 F F F 28 560-579 131 131 131 223 223 223 T T T 29 580-599 126 126 126 216 216 216 T T T Max gen: 132, Max flush: 225 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 132 132 132 225 225 225 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 133 133 133 226 226 226 F F F 4 080-099 132 132 132 225 225 225 F F F 5 100-119 127 127 127 219 219 219 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 132 132 132 225 225 225 F F F 8 160-179 128 128 128 220 220 220 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 132 132 132 225 225 225 F F F 11 220-239 132 132 132 225 225 225 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 132 132 132 225 225 225 F F F 14 280-299 132 132 132 225 225 225 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 131 131 131 223 223 223 F F F 17 340-359 133 133 133 226 226 226 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 128 128 128 220 220 220 F F F 20 400-419 131 131 131 223 223 223 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 131 131 131 223 223 223 F F F 24 480-499 133 133 133 226 226 226 F F F 25 500-519 133 133 133 226 226 226 F F F 26 520-539 131 131 131 223 223 223 F F F 27 540-559 132 132 132 225 225 225 F F F 28 560-579 133 133 133 226 226 226 F F F 29 580-599 133 133 133 226 226 226 F F F Max gen: 133, Max flush: 226 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.648819021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:24.691304512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.691715546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.723056239Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.723625954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.723668686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.723685652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.723703699Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726} {"msg":"Crucible 6903b389-5651-4efb-a7c0-51b7f3a4ffb9 has session id: 14c8f7ec-5598-4abd-bea8-21bb07b125e3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.724214535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Upstairs opts: Upstairs UUID: 6903b389-5651-4efb-a7c0-51b7f3a4ffb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.72424262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Crucible stats registered with UUID: 6903b389-5651-4efb-a7c0-51b7f3a4ffb9","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.724256827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:24.724274534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.724828911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.724855116Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726} {"msg":"6903b389-5651-4efb-a7c0-51b7f3a4ffb9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.724927133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.72494425Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"0","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.724969585Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"0","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.724983582Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"1","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725018326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"1","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725037002Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"2","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725061178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"2","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725084194Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.72510722Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725131655Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35413)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.72515598Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37521)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725394107Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63481)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725438348Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725841514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"1","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.725869439Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"2","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.726162294Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"0","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.726709123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"1","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.726737478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"2","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.726755315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"0","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727312921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727413683Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[0]R flush_numbers[0..12]: [226, 225, 222, 226, 225, 219, 220, 225, 220, 225, 225, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.72748436Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[0]R generation[0..12]: [133, 132, 130, 133, 132, 127, 128, 132, 128, 132, 132, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727505166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727519473Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[1]R flush_numbers[0..12]: [226, 225, 222, 226, 225, 219, 220, 225, 220, 225, 225, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727533021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[1]R generation[0..12]: [133, 132, 130, 133, 132, 127, 128, 132, 128, 132, 132, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727546958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727560886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[2]R flush_numbers[0..12]: [226, 225, 222, 226, 225, 219, 220, 225, 220, 225, 225, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727578902Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[2]R generation[0..12]: [133, 132, 130, 133, 132, 127, 128, 132, 128, 132, 132, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.72759257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727606237Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727625184Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727637951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727650499Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727662957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"":"downstairs","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727675734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727687422Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"0","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.72770021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"1","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727713087Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"client":"2","":"downstairs","session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727725855Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727737733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"6903b389-5651-4efb-a7c0-51b7f3a4ffb9 is now active with session: 14c8f7ec-5598-4abd-bea8-21bb07b125e3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727750161Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727761718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726,"session_id":"14c8f7ec-5598-4abd-bea8-21bb07b125e3"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:24.727776386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 32768 data: 7 6 5 6 7 8 8 11 02/30 Read block 463 len 40960 03/30 Read block 540 len 8192 04/30 Read block 90 len 16384 05/30 Read block 516 len 40960 06/30 Read block 211 len 8192 07/30 Write block 582 len 12288 data: 16 10 11 08/30 Read block 511 len 20480 09/30 Read block 35 len 28672 10/30 Write block 491 len 4096 data: 11 11/30 Write block 107 len 28672 data: 8 13 12 12 9 9 12 12/30 Read block 249 len 28672 13/30 Write block 530 len 32768 data: 8 9 8 6 5 5 7 11 14/30 Flush 15/30 Write block 578 len 12288 data: 13 14 17 16/30 Read block 490 len 8192 17/30 Write block 472 len 28672 data: 11 11 13 15 13 11 14 18/30 Read block 31 len 8192 19/30 Read block 472 len 36864 20/30 Read block 495 len 40960 21/30 Read block 514 len 36864 22/30 Read block 395 len 12288 23/30 Read block 117 len 20480 24/30 Read block 422 len 40960 25/30 Write block 386 len 32768 data: 10 9 9 11 11 12 12 11 26/30 Read block 62 len 16384 27/30 Read block 69 len 4096 28/30 Write block 564 len 28672 data: 15 16 14 13 13 9 9 29/30 Read block 201 len 32768 30/30 Write block 338 len 8192 data: 11 14 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 3 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:24.74937698Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 132 132 132 225 225 225 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 133 133 133 226 226 226 F F F 4 080-099 132 132 132 225 225 225 F F F 5 100-119 134 134 134 227 227 227 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 132 132 132 225 225 225 F F F 8 160-179 128 128 128 220 220 220 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 132 132 132 225 225 225 F F F 11 220-239 134 134 134 227 227 227 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 132 132 132 225 225 225 F F F 14 280-299 132 132 132 225 225 225 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 131 131 131 223 223 223 T T T 17 340-359 133 133 133 226 226 226 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 128 128 128 220 220 220 T T T 20 400-419 131 131 131 223 223 223 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 131 131 131 223 223 223 T T T 24 480-499 134 134 134 227 227 227 F F F 25 500-519 133 133 133 226 226 226 F F F 26 520-539 134 134 134 227 227 227 F F F 27 540-559 132 132 132 225 225 225 F F F 28 560-579 133 133 133 226 226 226 T T T 29 580-599 134 134 134 227 227 227 T T T Max gen: 134, Max flush: 227 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 132 132 132 225 225 225 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 133 133 133 226 226 226 F F F 4 080-099 132 132 132 225 225 225 F F F 5 100-119 134 134 134 227 227 227 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 132 132 132 225 225 225 F F F 8 160-179 128 128 128 220 220 220 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 132 132 132 225 225 225 F F F 11 220-239 134 134 134 227 227 227 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 132 132 132 225 225 225 F F F 14 280-299 132 132 132 225 225 225 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 133 133 133 226 226 226 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 135 135 135 228 228 228 F F F 20 400-419 131 131 131 223 223 223 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 135 135 135 228 228 228 F F F 24 480-499 134 134 134 227 227 227 F F F 25 500-519 133 133 133 226 226 226 F F F 26 520-539 134 134 134 227 227 227 F F F 27 540-559 132 132 132 225 225 225 F F F 28 560-579 135 135 135 228 228 228 F F F 29 580-599 135 135 135 228 228 228 F F F Max gen: 135, Max flush: 228 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.032509397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:29.077066555Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.077819475Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.110359096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.111519641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.111564273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.111578861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.111592878Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734} {"msg":"Crucible c57cd60c-9673-4499-ba3d-f2db14c4c16e has session id: 44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.112295718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Upstairs opts: Upstairs UUID: c57cd60c-9673-4499-ba3d-f2db14c4c16e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.112335321Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Crucible stats registered with UUID: c57cd60c-9673-4499-ba3d-f2db14c4c16e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.112351548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:29.112364265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.112842467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.112868932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734} {"msg":"c57cd60c-9673-4499-ba3d-f2db14c4c16e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.112949077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.112967894Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"0","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.113010406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"0","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.113026863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"1","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.11304119Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"1","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.113062546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"2","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.113077473Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"2","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.11309675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.113121995Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.11314783Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34782)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.113321278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34762)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.11336601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"ds_connection connected from Ok(127.0.0.1:53864)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.113394015Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.114262784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"1","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.114294178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"2","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.114309125Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"0","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.114962005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"1","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.114995099Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"0","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115190362Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"2","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115596307Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115723414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[0]R flush_numbers[0..12]: [226, 225, 222, 226, 225, 227, 220, 225, 220, 225, 225, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115804089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[0]R generation[0..12]: [133, 132, 130, 133, 132, 134, 128, 132, 128, 132, 132, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115825145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115839562Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[1]R flush_numbers[0..12]: [226, 225, 222, 226, 225, 227, 220, 225, 220, 225, 225, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.11585317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[1]R generation[0..12]: [133, 132, 130, 133, 132, 134, 128, 132, 128, 132, 132, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115867017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115880705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[2]R flush_numbers[0..12]: [226, 225, 222, 226, 225, 227, 220, 225, 220, 225, 225, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115894042Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[2]R generation[0..12]: [133, 132, 130, 133, 132, 134, 128, 132, 128, 132, 132, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.11590735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115921017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115933805Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115946863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:23:29.11595934Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115984846Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"":"downstairs","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.115997803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116009431Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"0","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116022668Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"1","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116035536Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"client":"2","":"downstairs","session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116048544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116060592Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"c57cd60c-9673-4499-ba3d-f2db14c4c16e is now active with session: 44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116072999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116084587Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734,"session_id":"44a0ebc6-8dcd-4b95-bfda-01f4a7be0ddb"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:29.116096325Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 192 len 32768 data: 11 11 13 10 11 10 9 8 02/30 Flush 03/30 Write block 157 len 4096 data: 13 04/30 Read block 375 len 24576 05/30 Write block 405 len 4096 data: 13 06/30 Read block 136 len 32768 07/30 Write block 204 len 24576 data: 12 11 13 13 9 10 08/30 Write block 269 len 20480 data: 8 8 8 11 8 09/30 Read block 574 len 36864 10/30 Write block 516 len 8192 data: 10 8 11/30 Read block 290 len 32768 12/30 Read block 16 len 28672 13/30 Write block 555 len 12288 data: 8 8 9 14/30 Write block 135 len 24576 data: 6 7 10 10 12 9 15/30 Write block 24 len 16384 data: 12 13 9 8 16/30 Write block 289 len 28672 data: 9 10 11 7 10 8 10 17/30 Read block 371 len 12288 18/30 Write block 584 len 24576 data: 12 9 7 7 8 9 19/30 Flush 20/30 Write block 58 len 8192 data: 6 6 21/30 Write block 205 len 40960 data: 12 14 14 10 11 9 10 9 8 7 22/30 Read block 167 len 40960 23/30 Read block 279 len 4096 24/30 Write block 589 len 8192 data: 10 7 25/30 Read block 207 len 16384 26/30 Read block 95 len 36864 27/30 Read block 506 len 4096 28/30 Write block 201 len 36864 data: 9 12 13 13 13 15 15 11 12 29/30 Write block 160 len 24576 data: 7 8 9 8 9 8 30/30 Read block 586 len 12288 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 6 Sent Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 9 9 11 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:29.136954956Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 130 130 130 222 222 222 T T T 3 060-079 133 133 133 226 226 226 F F F 4 080-099 132 132 132 225 225 225 F F F 5 100-119 134 134 134 227 227 227 F F F 6 120-139 136 136 136 230 230 230 F F F 7 140-159 136 136 136 230 230 230 F F F 8 160-179 128 128 128 220 220 220 T T T 9 180-199 136 136 136 229 229 229 F F F 10 200-219 136 136 136 230 230 230 T T T 11 220-239 134 134 134 227 227 227 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 136 136 136 230 230 230 F F F 14 280-299 136 136 136 230 230 230 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 133 133 133 226 226 226 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 135 135 135 228 228 228 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 135 135 135 228 228 228 F F F 24 480-499 134 134 134 227 227 227 F F F 25 500-519 136 136 136 230 230 230 F F F 26 520-539 134 134 134 227 227 227 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 135 135 135 228 228 228 F F F 29 580-599 136 136 136 230 230 230 T T T Max gen: 136, Max flush: 230 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 137 137 137 231 231 231 F F F 3 060-079 133 133 133 226 226 226 F F F 4 080-099 132 132 132 225 225 225 F F F 5 100-119 134 134 134 227 227 227 F F F 6 120-139 136 136 136 230 230 230 F F F 7 140-159 136 136 136 230 230 230 F F F 8 160-179 137 137 137 231 231 231 F F F 9 180-199 136 136 136 229 229 229 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 134 134 134 227 227 227 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 136 136 136 230 230 230 F F F 14 280-299 136 136 136 230 230 230 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 133 133 133 226 226 226 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 135 135 135 228 228 228 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 123 123 123 211 211 211 F F F 23 460-479 135 135 135 228 228 228 F F F 24 480-499 134 134 134 227 227 227 F F F 25 500-519 136 136 136 230 230 230 F F F 26 520-539 134 134 134 227 227 227 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 135 135 135 228 228 228 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 137, Max flush: 231 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.423762595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.470072348Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.471119914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.471166826Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.471181813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.471196131Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742} {"msg":"Crucible de25a62a-aaff-4b4a-bfbb-6872f216bbfa has session id: f0a81e0f-987f-4e13-9e8d-b45b648e2e8d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.471826654Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Upstairs opts: Upstairs UUID: de25a62a-aaff-4b4a-bfbb-6872f216bbfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.471879124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Crucible stats registered with UUID: de25a62a-aaff-4b4a-bfbb-6872f216bbfa","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47190359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:33.471924236Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472506928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472543591Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742} {"msg":"de25a62a-aaff-4b4a-bfbb-6872f216bbfa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472624376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472650662Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"0","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472690734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"0","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472718509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"1","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472744674Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"1","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47276954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"2","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472796464Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"2","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47281944Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472845146Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47287137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65436)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.472899825Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59863)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.473049748Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48086)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47319645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.473693149Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"2","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.473721363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"0","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.474243507Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"0","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.474349627Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"2","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.474892617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.474968053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.475606405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"1","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476161862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"1","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[0]R flush_numbers[0..12]: [226, 230, 231, 226, 225, 227, 230, 230, 231, 229, 231, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476748613Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[0]R generation[0..12]: [133, 136, 137, 133, 132, 134, 136, 136, 137, 136, 137, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476779078Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476797634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[1]R flush_numbers[0..12]: [226, 230, 231, 226, 225, 227, 230, 230, 231, 229, 231, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476814621Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[1]R generation[0..12]: [133, 136, 137, 133, 132, 134, 136, 136, 137, 136, 137, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476831918Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476849565Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[2]R flush_numbers[0..12]: [226, 230, 231, 226, 225, 227, 230, 230, 231, 229, 231, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47687354Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [133, 136, 137, 133, 132, 134, 136, 136, 137, 136, 137, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476897306Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476914963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47693109Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476947157Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476962714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.476978361Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"":"downstairs","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477014734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477031951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"0","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477048388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"1","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477064235Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"client":"2","":"downstairs","session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477080272Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477099068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"de25a62a-aaff-4b4a-bfbb-6872f216bbfa is now active with session: f0a81e0f-987f-4e13-9e8d-b45b648e2e8d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477114626Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.477129123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742,"session_id":"f0a81e0f-987f-4e13-9e8d-b45b648e2e8d"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:33.47714811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 239 len 4096 data: 12 02/30 Read block 548 len 36864 03/30 Write block 85 len 16384 data: 13 10 13 11 04/30 Write block 154 len 12288 data: 12 12 10 05/30 Read block 270 len 20480 06/30 Write block 435 len 28672 data: 9 11 11 11 11 13 12 07/30 Write block 65 len 24576 data: 12 10 9 11 16 20 08/30 Write block 151 len 24576 data: 12 12 12 13 13 11 09/30 Read block 301 len 24576 10/30 Read block 14 len 40960 11/30 Read block 276 len 12288 12/30 Read block 538 len 32768 13/30 Read block 526 len 16384 14/30 Flush 15/30 Write block 346 len 12288 data: 13 12 11 16/30 Read block 126 len 4096 17/30 Read block 71 len 24576 18/30 Write block 514 len 28672 data: 8 9 11 9 9 9 9 19/30 Read block 25 len 4096 20/30 Write block 382 len 32768 data: 9 10 11 9 11 10 10 12 21/30 Read block 514 len 16384 22/30 Flush 23/30 Write block 572 len 8192 data: 8 11 24/30 Write block 350 len 4096 data: 9 25/30 Read block 45 len 24576 26/30 Flush 27/30 Write block 173 len 20480 data: 12 13 13 13 13 28/30 Read block 273 len 4096 29/30 Write block 251 len 32768 data: 10 10 10 12 13 9 9 11 30/30 Write block 316 len 32768 data: 15 13 15 16 16 13 11 11 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:33.496936659Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 137 137 137 231 231 231 F F F 3 060-079 138 138 138 232 232 232 F F F 4 080-099 138 138 138 232 232 232 F F F 5 100-119 134 134 134 227 227 227 F F F 6 120-139 136 136 136 230 230 230 F F F 7 140-159 138 138 138 232 232 232 F F F 8 160-179 137 137 137 231 231 231 T T T 9 180-199 136 136 136 229 229 229 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 138 138 138 232 232 232 F F F 12 240-259 133 133 133 226 226 226 T T T 13 260-279 136 136 136 230 230 230 F F F 14 280-299 136 136 136 230 230 230 F F F 15 300-319 131 131 131 223 223 223 T T T 16 320-339 135 135 135 228 228 228 T T T 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 138 138 138 232 232 232 F F F 22 440-459 138 138 138 232 232 232 F F F 23 460-479 135 135 135 228 228 228 F F F 24 480-499 134 134 134 227 227 227 F F F 25 500-519 138 138 138 233 233 233 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 138 138 138 234 234 234 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 138, Max flush: 234 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 137 137 137 231 231 231 F F F 3 060-079 138 138 138 232 232 232 F F F 4 080-099 138 138 138 232 232 232 F F F 5 100-119 134 134 134 227 227 227 F F F 6 120-139 136 136 136 230 230 230 F F F 7 140-159 138 138 138 232 232 232 F F F 8 160-179 139 139 139 235 235 235 F F F 9 180-199 136 136 136 229 229 229 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 138 138 138 232 232 232 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 136 136 136 230 230 230 F F F 14 280-299 136 136 136 230 230 230 F F F 15 300-319 139 139 139 235 235 235 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 138 138 138 232 232 232 F F F 22 440-459 138 138 138 232 232 232 F F F 23 460-479 135 135 135 228 228 228 F F F 24 480-499 134 134 134 227 227 227 F F F 25 500-519 138 138 138 233 233 233 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 138 138 138 234 234 234 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 139, Max flush: 235 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.754164038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.800461444Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.801482735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.801528946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.801544494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.801559921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750} {"msg":"Crucible 985f833c-ab3d-49b0-b7d5-ece3af32f671 has session id: 6e323c7f-e75e-4109-be39-254d5fadfebd","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.802256222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Upstairs opts: Upstairs UUID: 985f833c-ab3d-49b0-b7d5-ece3af32f671, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.802297744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Crucible stats registered with UUID: 985f833c-ab3d-49b0-b7d5-ece3af32f671","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.802322459Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:37.802343446Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.802880806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.802905292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750} {"msg":"985f833c-ab3d-49b0-b7d5-ece3af32f671 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.802986237Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803026809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"0","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803053904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"0","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803067632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"1","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803081419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"1","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803094167Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"2","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803107914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"2","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803120502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.8031339Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803159305Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"ds_connection connected from Ok(127.0.0.1:55090)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803192459Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38479)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803341361Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"ds_connection connected from Ok(127.0.0.1:59470)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.803385293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.804085943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"2","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.804124386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"0","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.804842293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"0","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.804870388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"2","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.805201097Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"1","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.805442133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.805597624Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.805741847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"1","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[0]R flush_numbers[0..12]: [226, 230, 231, 232, 232, 227, 230, 232, 235, 229, 231, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806283967Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[0]R generation[0..12]: [133, 136, 137, 138, 138, 134, 136, 138, 139, 136, 137, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806305103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806325399Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[1]R flush_numbers[0..12]: [226, 230, 231, 232, 232, 227, 230, 232, 235, 229, 231, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806339787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[1]R generation[0..12]: [133, 136, 137, 138, 138, 134, 136, 138, 139, 136, 137, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806353254Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806367181Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[2]R flush_numbers[0..12]: [226, 230, 231, 232, 232, 227, 230, 232, 235, 229, 231, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806380959Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[2]R generation[0..12]: [133, 136, 137, 138, 138, 134, 136, 138, 139, 136, 137, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806394336Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806407754Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg"Wait for a query_work_queue command to finish before sending IO :"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.80642864Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806452926Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806475731Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806499007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"":"downstairs","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806523493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:23:37.806545069Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"0","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806569644Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"1","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806598748Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"client":"2","":"downstairs","session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806617725Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806630833Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"985f833c-ab3d-49b0-b7d5-ece3af32f671 is now active with session: 6e323c7f-e75e-4109-be39-254d5fadfebd","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.80664347Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806655278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750,"session_id":"6e323c7f-e75e-4109-be39-254d5fadfebd"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:37.806667146Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 543 len 24576 data: 13 13 14 14 13 14 02/30 Read block 81 len 12288 03/30 Read block 445 len 32768 04/30 Read block 489 len 36864 05/30 Flush 06/30 Write block 570 len 8192 data: 10 7 07/30 Write block 300 len 24576 data: 7 8 6 5 5 7 08/30 Write block 228 len 8192 data: 9 9 09/30 Read block 357 len 20480 10/30 Write block 479 len 20480 data: 14 15 13 13 13 11/30 Read block 389 len 32768 12/30 Write block 160 len 28672 data: 8 9 10 9 10 9 9 13/30 Write block 113 len 36864 data: 13 14 14 14 11 11 13 17 14 14/30 Read block 272 len 4096 15/30 Flush 16/30 Write block 507 len 32768 data: 10 7 8 8 9 8 9 9 17/30 Read block 44 len 24576 18/30 Write block 276 len 40960 data: 9 11 10 11 12 8 8 7 9 6 19/30 Write block 188 len 36864 data: 4 5 5 5 12 12 14 11 12 20/30 Read block 528 len 24576 21/30 Write block 108 len 40960 data: 14 13 13 10 10 14 15 15 15 12 22/30 Flush 23/30 Flush 24/30 Flush 25/30 Write block 115 len 28672 data: 16 16 13 12 14 18 15 26/30 Read block 473 len 40960 27/30 Write block 17 len 8192 data: 13 14 28/30 Read block 137 len 12288 29/30 Write block 304 len 36864 data: 6 8 8 9 12 10 12 14 15 30/30 Write block 422 len 20480 data: 9 12 15 10 11 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:37.828499637Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 T T T 1 020-039 136 136 136 230 230 230 F F F 2 040-059 137 137 137 231 231 231 F F F 3 060-079 138 138 138 232 232 232 F F F 4 080-099 138 138 138 232 232 232 F F F 5 100-119 140 140 140 238 238 238 T T T 6 120-139 140 140 140 237 237 237 T T T 7 140-159 138 138 138 232 232 232 F F F 8 160-179 140 140 140 237 237 237 F F F 9 180-199 140 140 140 238 238 238 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 140 140 140 237 237 237 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 140 140 140 238 238 238 F F F 14 280-299 140 140 140 238 238 238 F F F 15 300-319 140 140 140 237 237 237 T T T 16 320-339 139 139 139 235 235 235 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 138 138 138 232 232 232 T T T 22 440-459 138 138 138 232 232 232 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 140 140 140 237 237 237 F F F 25 500-519 140 140 140 238 238 238 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 140 140 140 236 236 236 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 140, Max flush: 238 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 137 137 137 231 231 231 F F F 3 060-079 138 138 138 232 232 232 F F F 4 080-099 138 138 138 232 232 232 F F F 5 100-119 141 141 141 239 239 239 F F F 6 120-139 141 141 141 239 239 239 F F F 7 140-159 138 138 138 232 232 232 F F F 8 160-179 140 140 140 237 237 237 F F F 9 180-199 140 140 140 238 238 238 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 140 140 140 237 237 237 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 140 140 140 238 238 238 F F F 14 280-299 140 140 140 238 238 238 F F F 15 300-319 141 141 141 239 239 239 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 141 141 141 239 239 239 F F F 22 440-459 138 138 138 232 232 232 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 140 140 140 237 237 237 F F F 25 500-519 140 140 140 238 238 238 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 140 140 140 236 236 236 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 141, Max flush: 239 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.087915312Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.134929785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.135899516Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.135945788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.135961685Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.135976182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758} {"msg":"Crucible 75364a90-e7f7-4970-91c0-36d5d31efc4b has session id: 4045c33c-6d56-4426-961b-1915a809aed3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.136643978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Upstairs opts: Upstairs UUID: 75364a90-e7f7-4970-91c0-36d5d31efc4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.136674643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Crucible stats registered with UUID: 75364a90-e7f7-4970-91c0-36d5d31efc4b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.13668943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:42.136701708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137212453Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137238258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758} {"msg":"75364a90-e7f7-4970-91c0-36d5d31efc4b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137297527Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137324692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"0","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137354807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"0","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137380462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"1","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137415056Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"1","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.13744709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"2","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137476134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"2","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.13750037Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137528075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.13755472Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46481)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137582195Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40636)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.13760555Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"ds_connection connected from Ok(127.0.0.1:59349)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.137631096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.138438406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"2","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.138477909Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"0","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.139008751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"0","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.139108192Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"2","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.139209784Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"1","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.139541902Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.13971905Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.139874331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"1","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[0]R flush_numbers[0..12]: [239, 230, 231, 232, 232, 239, 239, 232, 237, 238, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140413621Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[0]R generation[0..12]: [141, 136, 137, 138, 138, 141, 141, 138, 140, 140, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140438576Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140453284Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[1]R flush_numbers[0..12]: [239, 230, 231, 232, 232, 239, 239, 232, 237, 238, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140467131Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[1]R generation[0..12]: [141, 136, 137, 138, 138, 141, 141, 138, 140, 140, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140480609Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140494206Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[2]R flush_numbers[0..12]: [239, 230, 231, 232, 232, 239, 239, 232, 237, 238, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140507593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[2]R generation[0..12]: [141, 136, 137, 138, 138, 141, 141, 138, 140, 140, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140520881Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140534379Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:23:42.140546966Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140568472Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140586439Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140599627Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"":"downstairs","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140612314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140623942Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"0","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.14063691Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"1","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140650137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"client":"2","":"downstairs","session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140663255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140675162Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"75364a90-e7f7-4970-91c0-36d5d31efc4b is now active with session: 4045c33c-6d56-4426-961b-1915a809aed3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.14068751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140699138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758,"session_id":"4045c33c-6d56-4426-961b-1915a809aed3"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:42.140711026Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 446 len 4096 data: 10 02/30 Write block 40 len 4096 data: 11 03/30 Read block 343 len 8192 04/30 Read block 52 len 24576 05/30 Write block 257 len 8192 data: 10 12 06/30 Read block 523 len 4096 07/30 Write block 60 len 24576 data: 7 7 8 7 8 13 08/30 Read block 283 len 12288 09/30 Read block 215 len 24576 10/30 Read block 42 len 28672 11/30 Write block 180 len 24576 data: 15 13 13 9 10 10 12/30 Write block 427 len 20480 data: 8 7 9 8 9 13/30 Write block 95 len 8192 data: 9 7 14/30 Write block 483 len 40960 data: 14 16 14 15 14 13 9 11 12 12 15/30 Flush 16/30 Write block 474 len 28672 data: 14 16 14 12 15 15 16 17/30 Write block 273 len 20480 data: 9 7 7 10 12 18/30 Write block 554 len 4096 data: 8 19/30 Flush 20/30 Flush 21/30 Flush 22/30 Write block 482 len 4096 data: 14 23/30 Write block 483 len 4096 data: 15 24/30 Read block 31 len 24576 25/30 Read block 124 len 4096 26/30 Read block 337 len 20480 27/30 Read block 553 len 12288 28/30 Read block 92 len 20480 29/30 Read block 172 len 28672 30/30 Read block 101 len 36864 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:42.158300352Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 142 142 142 240 240 240 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 142 142 142 240 240 240 F F F 5 100-119 141 141 141 239 239 239 F F F 6 120-139 141 141 141 239 239 239 F F F 7 140-159 138 138 138 232 232 232 F F F 8 160-179 140 140 140 237 237 237 F F F 9 180-199 142 142 142 240 240 240 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 140 140 140 237 237 237 F F F 12 240-259 142 142 142 240 240 240 F F F 13 260-279 142 142 142 241 241 241 F F F 14 280-299 140 140 140 238 238 238 F F F 15 300-319 141 141 141 239 239 239 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 142 142 142 240 240 240 F F F 23 460-479 142 142 142 241 241 241 F F F 24 480-499 142 142 142 241 241 241 T T T 25 500-519 140 140 140 238 238 238 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 142 142 142 241 241 241 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 142, Max flush: 241 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 F F F 1 020-039 136 136 136 230 230 230 F F F 2 040-059 142 142 142 240 240 240 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 142 142 142 240 240 240 F F F 5 100-119 141 141 141 239 239 239 F F F 6 120-139 141 141 141 239 239 239 F F F 7 140-159 138 138 138 232 232 232 F F F 8 160-179 140 140 140 237 237 237 F F F 9 180-199 142 142 142 240 240 240 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 140 140 140 237 237 237 F F F 12 240-259 142 142 142 240 240 240 F F F 13 260-279 142 142 142 241 241 241 F F F 14 280-299 140 140 140 238 238 238 F F F 15 300-319 141 141 141 239 239 239 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 142 142 142 240 240 240 F F F 23 460-479 142 142 142 241 241 241 F F F 24 480-499 143 143 143 242 242 242 F F F 25 500-519 140 140 140 238 238 238 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 142 142 142 241 241 241 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 143, Max flush: 242 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.388867263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:46.434537625Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.435122137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.467678825Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.468723182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.468766514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.46878294Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.468800467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766} {"msg":"Crucible 9c2e81ad-9c52-4371-aab9-20284c2143e3 has session id: a96d76ec-0d6c-49ec-b438-9d80ead78062","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.469502727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Upstairs opts: Upstairs UUID: 9c2e81ad-9c52-4371-aab9-20284c2143e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.469534092Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Crucible stats registered with UUID: 9c2e81ad-9c52-4371-aab9-20284c2143e3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.469556177Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:46.469569005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470196859Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470221944Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766} {"msg":"9c2e81ad-9c52-4371-aab9-20284c2143e3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470279724Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.4702973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"0","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470322786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"0","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470336123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"1","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470350101Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"1","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470363048Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"2","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470376706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"2","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470389463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470402611Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470415809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"ds_connection connected from Ok(127.0.0.1:44419)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.47057114Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"ds_connection connected from Ok(127.0.0.1:51783)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470824763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"ds_connection connected from Ok(127.0.0.1:33959)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.470869954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.471256793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"2","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.471401716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"1","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.471564606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"0","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472014113Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"1","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.47213718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"0","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472212266Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"2","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472657044Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472772693Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[0]R flush_numbers[0..12]: [239, 230, 240, 240, 240, 239, 239, 232, 237, 240, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472848359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[0]R generation[0..12]: [141, 136, 142, 142, 142, 141, 141, 138, 140, 142, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472867215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472881242Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[1]R flush_numbers[0..12]: [239, 230, 240, 240, 240, 239, 239, 232, 237, 240, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.47289557Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[1]R generation[0..12]: [141, 136, 142, 142, 142, 141, 141, 138, 140, 142, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472909547Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472927994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[2]R flush_numbers[0..12]: [239, 230, 240, 240, 240, 239, 239, 232, 237, 240, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472941991Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[2]R generation[0..12]: [141, 136, 142, 142, 142, 141, 141, 138, 140, 142, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472955369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472969096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.472981594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:23:46.472996451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473034374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473049251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"":"downstairs","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473062359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473074077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"0","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473087324Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"1","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473099952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"client":"2","":"downstairs","session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.47311273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473124657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"9c2e81ad-9c52-4371-aab9-20284c2143e3 is now active with session: a96d76ec-0d6c-49ec-b438-9d80ead78062","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473137185Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.473151652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766,"session_id":"a96d76ec-0d6c-49ec-b438-9d80ead78062"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:46.47316384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 240 len 32768 data: 9 8 7 8 6 6 9 10 02/30 Write block 159 len 16384 data: 7 9 10 11 03/30 Write block 283 len 36864 data: 8 10 7 6 5 7 10 11 12 04/30 Read block 308 len 24576 05/30 Write block 254 len 12288 data: 13 14 10 06/30 Write block 446 len 16384 data: 11 9 6 9 07/30 Read block 295 len 36864 08/30 Write block 193 len 16384 data: 13 15 12 13 09/30 Write block 494 len 12288 data: 11 12 12 10/30 Read block 233 len 16384 11/30 Write block 281 len 12288 data: 9 9 9 12/30 Read block 559 len 28672 13/30 Read block 174 len 8192 14/30 Write block 181 len 4096 data: 14 15/30 Read block 347 len 8192 16/30 Write block 485 len 16384 data: 15 16 15 14 17/30 Write block 178 len 28672 data: 12 15 16 15 14 10 11 18/30 Write block 20 len 20480 data: 13 15 14 15 13 19/30 Read block 481 len 40960 20/30 Write block 268 len 28672 data: 7 9 9 9 12 10 8 21/30 Read block 359 len 16384 22/30 Write block 305 len 40960 data: 9 9 10 13 11 13 15 16 18 16 23/30 Read block 377 len 12288 24/30 Write block 261 len 12288 data: 13 11 8 25/30 Read block 329 len 8192 26/30 Write block 118 len 12288 data: 13 15 19 27/30 Read block 6 len 28672 28/30 Write block 60 len 28672 data: 8 8 9 8 9 14 11 29/30 Read block 448 len 36864 30/30 Write block 482 len 20480 data: 15 16 17 16 17 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:46.492812626Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 F F F 1 020-039 136 136 136 230 230 230 T T T 2 040-059 142 142 142 240 240 240 F F F 3 060-079 142 142 142 240 240 240 T T T 4 080-099 142 142 142 240 240 240 F F F 5 100-119 141 141 141 239 239 239 T T T 6 120-139 141 141 141 239 239 239 T T T 7 140-159 138 138 138 232 232 232 T T T 8 160-179 140 140 140 237 237 237 T T T 9 180-199 142 142 142 240 240 240 T T T 10 200-219 137 137 137 231 231 231 F F F 11 220-239 140 140 140 237 237 237 F F F 12 240-259 142 142 142 240 240 240 T T T 13 260-279 142 142 142 241 241 241 T T T 14 280-299 140 140 140 238 238 238 T T T 15 300-319 141 141 141 239 239 239 T T T 16 320-339 139 139 139 235 235 235 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 142 142 142 240 240 240 T T T 23 460-479 142 142 142 241 241 241 F F F 24 480-499 143 143 143 242 242 242 T T T 25 500-519 140 140 140 238 238 238 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 142 142 142 241 241 241 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 143, Max flush: 242 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 F F F 1 020-039 144 144 144 243 243 243 F F F 2 040-059 142 142 142 240 240 240 F F F 3 060-079 144 144 144 243 243 243 F F F 4 080-099 142 142 142 240 240 240 F F F 5 100-119 144 144 144 243 243 243 F F F 6 120-139 144 144 144 243 243 243 F F F 7 140-159 144 144 144 243 243 243 F F F 8 160-179 144 144 144 243 243 243 F F F 9 180-199 144 144 144 243 243 243 F F F 10 200-219 137 137 137 231 231 231 F F F 11 220-239 140 140 140 237 237 237 F F F 12 240-259 144 144 144 243 243 243 F F F 13 260-279 144 144 144 243 243 243 F F F 14 280-299 144 144 144 243 243 243 F F F 15 300-319 144 144 144 243 243 243 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 130 130 130 221 221 221 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 144 144 144 243 243 243 F F F 23 460-479 142 142 142 241 241 241 F F F 24 480-499 144 144 144 243 243 243 F F F 25 500-519 140 140 140 238 238 238 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 142 142 142 241 241 241 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 144, Max flush: 243 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.823085733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.869716367Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.870745327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.870789959Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.870804426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.870819083Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774} {"msg":"Crucible 11a2e709-e4b3-43a9-a10c-bb9ca4647d0a has session id: d0e1a036-a82c-4f9a-adfd-8aca336ae8cf","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.871500217Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Upstairs opts: Upstairs UUID: 11a2e709-e4b3-43a9-a10c-bb9ca4647d0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.871530202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Crucible stats registered with UUID: 11a2e709-e4b3-43a9-a10c-bb9ca4647d0a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.871545149Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:50.871564055Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872158175Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872191689Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774} {"msg":"11a2e709-e4b3-43a9-a10c-bb9ca4647d0a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872291111Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872324895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"0","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872389972Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"0","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872423186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"1","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872453681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"1","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872475487Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"2","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872503921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"2","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872534946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872561631Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872575958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58772)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872841249Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:36226)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872909266Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:37271)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.872948309Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.87375654Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"0","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.873786274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"1","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.874415608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"2","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.874545534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"1","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.874569679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"0","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.874957928Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"2","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875241255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875265911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[0]R flush_numbers[0..12]: [239, 243, 240, 243, 240, 243, 243, 243, 243, 243, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875524493Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[0]R generation[0..12]: [141, 144, 142, 144, 142, 144, 144, 144, 144, 144, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875561226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875587691Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[1]R flush_numbers[0..12]: [239, 243, 240, 243, 240, 243, 243, 243, 243, 243, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875614186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[1]R generation[0..12]: [141, 144, 142, 144, 142, 144, 144, 144, 144, 144, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875638831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875661847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[2]R flush_numbers[0..12]: [239, 243, 240, 243, 240, 243, 243, 243, 243, 243, 231, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875687702Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[2]R generation[0..12]: [141, 144, 142, 144, 142, 144, 144, 144, 144, 144, 137, 140]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:23:50.875714737Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875747941Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Max found gen is 145","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875780575Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Generation requested: 146 >= found:145","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.87580768Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875832915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875859431Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"":"downstairs","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875885886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:23:50.875911001Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"0","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875940326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"1","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875965201Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"client":"2","":"downstairs","session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875982478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.875997085Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"11a2e709-e4b3-43a9-a10c-bb9ca4647d0a is now active with session: d0e1a036-a82c-4f9a-adfd-8aca336ae8cf","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.876020381Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.876036608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774,"session_id":"d0e1a036-a82c-4f9a-adfd-8aca336ae8cf"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:50.876049226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 24576 data: 10 11 10 9 8 5 02/30 Write block 47 len 4096 data: 8 03/30 Read block 302 len 8192 04/30 Read block 429 len 32768 05/30 Write block 317 len 16384 data: 14 16 17 17 06/30 Write block 163 len 16384 data: 10 11 10 10 07/30 Read block 313 len 16384 08/30 Read block 287 len 20480 09/30 Write block 250 len 12288 data: 11 11 11 10/30 Read block 550 len 4096 11/30 Read block 248 len 36864 12/30 Read block 42 len 20480 13/30 Write block 248 len 40960 data: 11 11 12 12 12 11 14 15 11 11 14/30 Read block 156 len 20480 15/30 Read block 256 len 20480 16/30 Read block 459 len 28672 17/30 Read block 221 len 36864 18/30 Write block 239 len 20480 data: 13 10 9 8 9 19/30 Flush 20/30 Write block 84 len 8192 data: 12 14 21/30 Read block 72 len 28672 22/30 Write block 368 len 12288 data: 13 14 12 23/30 Flush 24/30 Flush 25/30 Write block 218 len 20480 data: 8 8 7 5 7 26/30 Write block 135 len 32768 data: 7 8 11 11 13 10 8 11 27/30 Read block 168 len 20480 28/30 Read block 67 len 28672 29/30 Read block 320 len 28672 30/30 Write block 499 len 40960 data: 11 11 7 9 12 11 13 11 11 8 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:50.897352365Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 F F F 1 020-039 144 144 144 243 243 243 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 144 144 144 243 243 243 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 144 144 144 243 243 243 F F F 6 120-139 144 144 144 243 243 243 T T T 7 140-159 144 144 144 243 243 243 T T T 8 160-179 146 146 146 244 244 244 F F F 9 180-199 144 144 144 243 243 243 F F F 10 200-219 146 146 146 244 244 244 T T T 11 220-239 146 146 146 244 244 244 T T T 12 240-259 146 146 146 244 244 244 F F F 13 260-279 144 144 144 243 243 243 F F F 14 280-299 144 144 144 243 243 243 F F F 15 300-319 146 146 146 244 244 244 F F F 16 320-339 146 146 146 244 244 244 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 146 146 146 245 245 245 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 144 144 144 243 243 243 F F F 23 460-479 142 142 142 241 241 241 F F F 24 480-499 144 144 144 243 243 243 T T T 25 500-519 140 140 140 238 238 238 T T T 26 520-539 138 138 138 233 233 233 F F F 27 540-559 142 142 142 241 241 241 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 146, Max flush: 245 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 F F F 1 020-039 144 144 144 243 243 243 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 144 144 144 243 243 243 F F F 4 080-099 146 146 146 245 245 245 F F F 5 100-119 144 144 144 243 243 243 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 144 144 144 243 243 243 F F F 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 146 146 146 244 244 244 F F F 13 260-279 144 144 144 243 243 243 F F F 14 280-299 144 144 144 243 243 243 F F F 15 300-319 146 146 146 244 244 244 F F F 16 320-339 146 146 146 244 244 244 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 146 146 146 245 245 245 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 144 144 144 243 243 243 F F F 23 460-479 142 142 142 241 241 241 F F F 24 480-499 147 147 147 246 246 246 F F F 25 500-519 147 147 147 246 246 246 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 142 142 142 241 241 241 F F F 28 560-579 140 140 140 237 237 237 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 147, Max flush: 246 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.156693453Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.201802839Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.202752304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.202805514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.20282326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.202841837Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782} {"msg":"Crucible b5d63125-71c2-4e37-b668-fe06aa83f4d5 has session id: bbc55f12-901a-4874-840e-e5de395753cf","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.203388376Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Upstairs opts: Upstairs UUID: b5d63125-71c2-4e37-b668-fe06aa83f4d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.203438926Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Crucible stats registered with UUID: b5d63125-71c2-4e37-b668-fe06aa83f4d5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.20347266Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:55.203498655Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204030167Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204058742Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782} {"msg":"b5d63125-71c2-4e37-b668-fe06aa83f4d5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204134678Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204155084Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"0","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204188948Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"0","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204215473Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"1","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204245717Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"1","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204272512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204301287Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204330101Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204360226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204389321Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:50403)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204465127Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:55.204510958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:55.204549451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:37779)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204637275Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:23:55.204728068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.204764821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.205280096Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"1","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.205321948Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"0","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.205999923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"1","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.206040865Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"0","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.206665669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:23:55.206698734Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.205697808Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:63094)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.205799669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.206999148Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.207553705Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[0]R flush_numbers[0..12]: [239, 243, 244, 243, 245, 243, 246, 246, 244, 243, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208166072Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[0]R generation[0..12]: [141, 144, 146, 144, 146, 144, 147, 147, 146, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208190487Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208205835Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[1]R flush_numbers[0..12]: [239, 243, 244, 243, 245, 243, 246, 246, 244, 243, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208220072Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[1]R generation[0..12]: [141, 144, 146, 144, 146, 144, 147, 147, 146, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208233819Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208247807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[2]R flush_numbers[0..12]: [239, 243, 244, 243, 245, 243, 246, 246, 244, 243, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208261684Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[2]R generation[0..12]: [141, 144, 146, 144, 146, 144, 147, 147, 146, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208275282Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208289279Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:24:05.208301957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208318304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208331191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208343849Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"":"downstairs","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208360396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208372634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"0","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208385701Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"1","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208398609Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"client":"2","":"downstairs","session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208411366Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208423374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"b5d63125-71c2-4e37-b668-fe06aa83f4d5 is now active with session: bbc55f12-901a-4874-840e-e5de395753cf","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208436032Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.20844774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782,"session_id":"bbc55f12-901a-4874-840e-e5de395753cf"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:05.208460068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 40960 data: 13 13 12 15 16 12 12 13 12 10 02/30 Read block 343 len 12288 03/30 Read block 352 len 40960 04/30 Read block 305 len 16384 05/30 Write block 251 len 12288 data: 14 14 13 06/30 Write block 371 len 16384 data: 14 15 12 13 07/30 Read block 306 len 8192 08/30 Write block 152 len 20480 data: 13 13 14 14 12 09/30 Write block 77 len 16384 data: 8 8 9 10 10/30 Read block 105 len 20480 11/30 Write block 544 len 20480 data: 14 15 15 14 15 12/30 Write block 452 len 36864 data: 8 11 10 11 11 12 11 10 10 13/30 Read block 195 len 4096 14/30 Write block 77 len 20480 data: 9 9 10 11 10 15/30 Write block 572 len 4096 data: 9 16/30 Read block 374 len 20480 17/30 Read block 100 len 12288 18/30 Read block 223 len 40960 19/30 Write block 101 len 12288 data: 8 8 7 20/30 Read block 397 len 4096 21/30 Read block 221 len 36864 22/30 Read block 300 len 20480 23/30 Read block 78 len 36864 24/30 Write block 79 len 28672 data: 11 12 11 12 10 13 15 25/30 Read block 259 len 28672 26/30 Read block 344 len 28672 27/30 Write block 19 len 40960 data: 13 14 16 15 16 14 14 10 9 7 28/30 Write block 411 len 24576 data: 10 10 10 11 9 9 29/30 Write block 492 len 36864 data: 13 11 12 13 13 11 10 12 12 30/30 Read block 390 len 36864 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 28 30 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:05.229835553Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 239 239 239 T T T 1 020-039 144 144 144 243 243 243 T T T 2 040-059 146 146 146 244 244 244 F F F 3 060-079 144 144 144 243 243 243 T T T 4 080-099 146 146 146 245 245 245 T T T 5 100-119 144 144 144 243 243 243 T T T 6 120-139 147 147 147 246 246 246 F F F 7 140-159 147 147 147 246 246 246 T T T 8 160-179 146 146 146 244 244 244 F F F 9 180-199 144 144 144 243 243 243 F F F 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 146 146 146 244 244 244 T T T 13 260-279 144 144 144 243 243 243 T T T 14 280-299 144 144 144 243 243 243 F F F 15 300-319 146 146 146 244 244 244 F F F 16 320-339 146 146 146 244 244 244 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 146 146 146 245 245 245 T T T 19 380-399 138 138 138 233 233 233 F F F 20 400-419 136 136 136 230 230 230 T T T 21 420-439 142 142 142 240 240 240 F F F 22 440-459 144 144 144 243 243 243 T T T 23 460-479 142 142 142 241 241 241 T T T 24 480-499 147 147 147 246 246 246 T T T 25 500-519 147 147 147 246 246 246 T T T 26 520-539 138 138 138 233 233 233 F F F 27 540-559 142 142 142 241 241 241 T T T 28 560-579 140 140 140 237 237 237 T T T 29 580-599 137 137 137 231 231 231 F F F Max gen: 147, Max flush: 246 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 148 148 148 247 247 247 F F F 4 080-099 148 148 148 247 247 247 F F F 5 100-119 148 148 148 247 247 247 F F F 6 120-139 147 147 147 246 246 246 F F F 7 140-159 148 148 148 247 247 247 F F F 8 160-179 146 146 146 244 244 244 F F F 9 180-199 144 144 144 243 243 243 F F F 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 148 148 148 247 247 247 F F F 13 260-279 148 148 148 247 247 247 F F F 14 280-299 144 144 144 243 243 243 F F F 15 300-319 146 146 146 244 244 244 F F F 16 320-339 146 146 146 244 244 244 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 148 148 148 247 247 247 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 148 148 148 247 247 247 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 148 148 148 247 247 247 F F F 23 460-479 148 148 148 247 247 247 F F F 24 480-499 148 148 148 247 247 247 F F F 25 500-519 148 148 148 247 247 247 F F F 26 520-539 138 138 138 233 233 233 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 148 148 148 247 247 247 F F F 29 580-599 137 137 137 231 231 231 F F F Max gen: 148, Max flush: 247 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.612726948Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:19.668159684Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.669604797Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.705378639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.706924793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.706981423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.706999709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.707017736Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790} {"msg":"Crucible adb256ff-f5be-41e0-865c-2ebf203c02f3 has session id: ba025b1c-e3a0-4f80-a07e-a005fe2b9c03","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.707837704Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Upstairs opts: Upstairs UUID: adb256ff-f5be-41e0-865c-2ebf203c02f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.707880806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Crucible stats registered with UUID: adb256ff-f5be-41e0-865c-2ebf203c02f3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.707899533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:19.70791558Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708594324Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708625399Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790} {"msg":"adb256ff-f5be-41e0-865c-2ebf203c02f3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708710773Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708733688Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"0","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.70877981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"0","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708806895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"1","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.70883414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"1","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708859355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"2","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708884021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"2","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708907346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708933171Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.708968145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:60702)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.709166068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:49832)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.709328738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"ds_connection connected from Ok(127.0.0.1:32821)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.709354813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.709974839Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"1","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.710008073Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"2","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.710468238Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"0","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.710754994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"2","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.710928733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"1","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.711286956Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"0","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.711596189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.711711757Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[0]R flush_numbers[0..12]: [247, 247, 244, 247, 247, 247, 246, 247, 244, 243, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.71212812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[0]R generation[0..12]: [148, 148, 146, 148, 148, 148, 147, 148, 146, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712148497Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712162994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[1]R flush_numbers[0..12]: [247, 247, 244, 247, 247, 247, 246, 247, 244, 243, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.7121855Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[1]R generation[0..12]: [148, 148, 146, 148, 148, 148, 147, 148, 146, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712200607Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712214744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[2]R flush_numbers[0..12]: [247, 247, 244, 247, 247, 247, 246, 247, 244, 243, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712228042Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[2]R generation[0..12]: [148, 148, 146, 148, 148, 148, 147, 148, 146, 144, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.71224191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712256497Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Max found gen is 149","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712269654Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Generation requested: 150 >= found:149","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712283022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.71229568Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712308997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"":"downstairs","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712322305Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712342301Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"0","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712356128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"1","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712369726Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"client":"2","":"downstairs","session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712388662Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.71240141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"adb256ff-f5be-41e0-865c-2ebf203c02f3 is now active with session: ba025b1c-e3a0-4f80-a07e-a005fe2b9c03","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712414687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712426375Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790,"session_id":"ba025b1c-e3a0-4f80-a07e-a005fe2b9c03"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:19.712438293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 255 len 40960 data: 17 13 13 14 13 11 14 12 9 7 02/30 Flush 03/30 Write block 119 len 8192 data: 16 20 04/30 Flush 05/30 Write block 579 len 36864 data: 15 18 17 17 11 13 10 8 8 06/30 Read block 476 len 16384 07/30 Read block 84 len 32768 08/30 Write block 183 len 4096 data: 11 09/30 Read block 371 len 28672 10/30 Read block 423 len 32768 11/30 Write block 282 len 24576 data: 10 10 11 8 7 6 12/30 Read block 537 len 12288 13/30 Read block 40 len 40960 14/30 Write block 171 len 36864 data: 11 13 13 14 14 14 14 13 16 15/30 Read block 510 len 8192 16/30 Read block 351 len 40960 17/30 Write block 368 len 28672 data: 14 15 13 15 16 13 14 18/30 Write block 253 len 32768 data: 14 16 18 14 14 15 14 12 19/30 Write block 62 len 8192 data: 10 9 20/30 Read block 395 len 8192 21/30 Read block 296 len 32768 22/30 Read block 523 len 12288 23/30 Read block 58 len 32768 24/30 Read block 315 len 24576 25/30 Read block 205 len 20480 26/30 Write block 310 len 12288 data: 14 16 17 27/30 Write block 517 len 40960 data: 10 10 10 10 8 9 9 12 11 8 28/30 Read block 433 len 4096 29/30 Write block 312 len 36864 data: 18 19 17 14 16 15 17 18 18 30/30 Read block 524 len 36864 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 9 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 26 25 25 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:19.737571954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 148 148 148 247 247 247 T T T 4 080-099 148 148 148 247 247 247 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 150 150 150 249 249 249 F F F 7 140-159 148 148 148 247 247 247 F F F 8 160-179 146 146 146 244 244 244 T T T 9 180-199 144 144 144 243 243 243 T T T 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 150 150 150 248 248 248 T T T 13 260-279 150 150 150 248 248 248 T T T 14 280-299 144 144 144 243 243 243 T T T 15 300-319 146 146 146 244 244 244 T T T 16 320-339 146 146 146 244 244 244 T T T 17 340-359 138 138 138 234 234 234 F F F 18 360-379 148 148 148 247 247 247 T T T 19 380-399 138 138 138 233 233 233 F F F 20 400-419 148 148 148 247 247 247 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 148 148 148 247 247 247 F F F 23 460-479 148 148 148 247 247 247 F F F 24 480-499 148 148 148 247 247 247 F F F 25 500-519 148 148 148 247 247 247 T T T 26 520-539 138 138 138 233 233 233 T T T 27 540-559 148 148 148 247 247 247 F F F 28 560-579 148 148 148 247 247 247 T T T 29 580-599 137 137 137 231 231 231 T T T Max gen: 150, Max flush: 249 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 146 146 146 244 244 244 F F F 3 060-079 151 151 151 250 250 250 F F F 4 080-099 148 148 148 247 247 247 F F F 5 100-119 150 150 150 249 249 249 F F F 6 120-139 150 150 150 249 249 249 F F F 7 140-159 148 148 148 247 247 247 F F F 8 160-179 151 151 151 250 250 250 F F F 9 180-199 151 151 151 250 250 250 F F F 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 151 151 151 250 250 250 F F F 13 260-279 151 151 151 250 250 250 F F F 14 280-299 151 151 151 250 250 250 F F F 15 300-319 151 151 151 250 250 250 F F F 16 320-339 151 151 151 250 250 250 F F F 17 340-359 138 138 138 234 234 234 F F F 18 360-379 151 151 151 250 250 250 F F F 19 380-399 138 138 138 233 233 233 F F F 20 400-419 148 148 148 247 247 247 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 148 148 148 247 247 247 F F F 23 460-479 148 148 148 247 247 247 F F F 24 480-499 148 148 148 247 247 247 F F F 25 500-519 151 151 151 250 250 250 F F F 26 520-539 151 151 151 250 250 250 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 151 151 151 250 250 250 F F F 29 580-599 151 151 151 250 250 250 F F F Max gen: 151, Max flush: 250 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.105476989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.150508999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.151243404Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.151292845Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.151310751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.151329848Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798} {"msg":"Crucible 94993a9e-3190-44fa-8c67-d09328647af4 has session id: d4ef36a2-8937-4830-8c06-86de468121a3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.151938265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Upstairs opts: Upstairs UUID: 94993a9e-3190-44fa-8c67-d09328647af4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.151977978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Crucible stats registered with UUID: 94993a9e-3190-44fa-8c67-d09328647af4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152001593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:24.152023509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152627778Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152654273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798} {"msg":"94993a9e-3190-44fa-8c67-d09328647af4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152742856Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152761693Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"0","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152787688Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"0","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152806844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"1","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15283215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"1","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152858785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"2","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152888299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"2","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152913515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15293847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.152965535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46033)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15299277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43983)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.153018065Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39284)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.153038451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.153620014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"2","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.153796761Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"0","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.154267564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"2","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.154435973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"0","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.154591564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"1","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15477849Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.154972624Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155244154Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"1","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[0]R flush_numbers[0..12]: [247, 247, 244, 250, 247, 249, 249, 247, 250, 250, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15575054Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[0]R generation[0..12]: [148, 148, 146, 151, 148, 150, 150, 148, 151, 151, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155775095Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155790222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[1]R flush_numbers[0..12]: [247, 247, 244, 250, 247, 249, 249, 247, 250, 250, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15580384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[1]R generation[0..12]: [148, 148, 146, 151, 148, 150, 150, 148, 151, 151, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155817417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155831265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[2]R flush_numbers[0..12]: [247, 247, 244, 250, 247, 249, 249, 247, 250, 250, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155844932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[2]R generation[0..12]: [148, 148, 146, 151, 148, 150, 150, 148, 151, 151, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15585824Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155876637Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155896183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155909451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155922108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155934746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"":"downstairs","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155947424Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155958971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"0","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155971899Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"1","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155984617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"client":"2","":"downstairs","session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.155997464Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.156009392Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"94993a9e-3190-44fa-8c67-d09328647af4 is now active with session: d4ef36a2-8937-4830-8c06-86de468121a3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.15602189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.156033408Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798,"session_id":"d4ef36a2-8937-4830-8c06-86de468121a3"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:24.156045305Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 357 len 20480 data: 17 18 19 17 15 02/30 Read block 165 len 36864 03/30 Read block 152 len 24576 04/30 Write block 522 len 28672 data: 10 10 13 12 9 8 8 05/30 Write block 74 len 32768 data: 20 15 13 10 10 12 13 12 06/30 Write block 361 len 16384 data: 16 12 11 11 07/30 Flush 08/30 Write block 380 len 40960 data: 10 9 10 11 12 10 12 11 11 13 09/30 Read block 407 len 4096 10/30 Write block 132 len 20480 data: 4 7 4 8 9 11/30 Write block 102 len 24576 data: 9 8 6 7 7 9 12/30 Read block 158 len 40960 13/30 Write block 347 len 36864 data: 13 12 11 10 8 8 11 11 13 14/30 Read block 467 len 24576 15/30 Write block 384 len 36864 data: 13 11 13 12 12 14 12 13 13 16/30 Read block 342 len 24576 17/30 Write block 587 len 4096 data: 9 18/30 Write block 186 len 40960 data: 9 7 5 6 6 6 13 14 16 13 19/30 Write block 243 len 32768 data: 10 7 7 10 11 12 12 13 20/30 Write block 156 len 16384 data: 13 14 12 8 21/30 Write block 290 len 16384 data: 12 13 8 11 22/30 Write block 46 len 4096 data: 10 23/30 Write block 288 len 20480 data: 8 11 13 14 9 24/30 Write block 104 len 36864 data: 7 8 8 10 15 14 14 11 11 25/30 Write block 186 len 24576 data: 10 8 6 7 7 7 26/30 Write block 308 len 12288 data: 14 12 15 27/30 Read block 27 len 8192 28/30 Read block 390 len 36864 29/30 Read block 22 len 20480 30/30 Write block 574 len 32768 data: 12 12 10 12 14 16 19 18 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 10 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:24.176198258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 146 146 146 244 244 244 T T T 3 060-079 152 152 152 251 251 251 F F F 4 080-099 152 152 152 251 251 251 F F F 5 100-119 150 150 150 249 249 249 T T T 6 120-139 150 150 150 249 249 249 T T T 7 140-159 148 148 148 247 247 247 T T T 8 160-179 151 151 151 250 250 250 F F F 9 180-199 151 151 151 250 250 250 T T T 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 151 151 151 250 250 250 T T T 13 260-279 151 151 151 250 250 250 F F F 14 280-299 151 151 151 250 250 250 T T T 15 300-319 151 151 151 250 250 250 T T T 16 320-339 151 151 151 250 250 250 F F F 17 340-359 152 152 152 251 251 251 T T T 18 360-379 152 152 152 251 251 251 F F F 19 380-399 138 138 138 233 233 233 T T T 20 400-419 148 148 148 247 247 247 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 148 148 148 247 247 247 F F F 23 460-479 148 148 148 247 247 247 F F F 24 480-499 148 148 148 247 247 247 F F F 25 500-519 151 151 151 250 250 250 F F F 26 520-539 152 152 152 251 251 251 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 151 151 151 250 250 250 T T T 29 580-599 151 151 151 250 250 250 T T T Max gen: 152, Max flush: 251 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 152 152 152 251 251 251 F F F 4 080-099 152 152 152 251 251 251 F F F 5 100-119 153 153 153 252 252 252 F F F 6 120-139 153 153 153 252 252 252 F F F 7 140-159 153 153 153 252 252 252 F F F 8 160-179 151 151 151 250 250 250 F F F 9 180-199 153 153 153 252 252 252 F F F 10 200-219 147 147 147 246 246 246 F F F 11 220-239 147 147 147 246 246 246 F F F 12 240-259 153 153 153 252 252 252 F F F 13 260-279 151 151 151 250 250 250 F F F 14 280-299 153 153 153 252 252 252 F F F 15 300-319 153 153 153 252 252 252 F F F 16 320-339 151 151 151 250 250 250 F F F 17 340-359 153 153 153 252 252 252 F F F 18 360-379 152 152 152 251 251 251 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 148 148 148 247 247 247 F F F 21 420-439 142 142 142 240 240 240 F F F 22 440-459 148 148 148 247 247 247 F F F 23 460-479 148 148 148 247 247 247 F F F 24 480-499 148 148 148 247 247 247 F F F 25 500-519 151 151 151 250 250 250 F F F 26 520-539 152 152 152 251 251 251 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 153 153 153 252 252 252 F F F 29 580-599 153 153 153 252 252 252 F F F Max gen: 153, Max flush: 252 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.468942656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.514137365Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.515094059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.51514104Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.515164066Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.515178923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806} {"msg":"Crucible 6b361b13-adb8-4282-bb29-3464ef1e0b75 has session id: b3006d94-13ee-44da-a656-98565482ec3a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.515697817Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Upstairs opts: Upstairs UUID: 6b361b13-adb8-4282-bb29-3464ef1e0b75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.51573732Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Crucible stats registered with UUID: 6b361b13-adb8-4282-bb29-3464ef1e0b75","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.515760815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:38.515781291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516249315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.51627385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806} {"msg":"6b361b13-adb8-4282-bb29-3464ef1e0b75 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516346327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516364053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"0","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516389209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"0","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516403046Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"1","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516416914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"1","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516429512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516447578Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516460746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516473893Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516496339Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:38.516631874Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64660)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516672387Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:38.516704481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46388)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516733735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:38.516788615Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.516915622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.517382355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"0","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.5174103Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"1","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.517965157Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"1","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.518086245Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"0","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.518531863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:38.518629385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.517980694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64064)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.518246515Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.519602024Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.520293486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[0]R flush_numbers[0..12]: [247, 247, 252, 251, 251, 252, 252, 252, 250, 252, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.520960753Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[0]R generation[0..12]: [148, 148, 153, 152, 152, 153, 153, 153, 151, 153, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.520988198Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521003745Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[1]R flush_numbers[0..12]: [247, 247, 252, 251, 251, 252, 252, 252, 250, 252, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521017692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[1]R generation[0..12]: [148, 148, 153, 152, 152, 153, 153, 153, 151, 153, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.5210313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521045127Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[2]R flush_numbers[0..12]: [247, 247, 252, 251, 251, 252, 252, 252, 250, 252, 246, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521058675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[2]R generation[0..12]: [148, 148, 153, 152, 152, 153, 153, 153, 151, 153, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521079081Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521093398Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:24:48.521106226Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521124852Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.52113726Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521149818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"":"downstairs","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521162645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521174153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"0","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521187091Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"1","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521199809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"client":"2","":"downstairs","session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521212456Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521224374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"6b361b13-adb8-4282-bb29-3464ef1e0b75 is now active with session: b3006d94-13ee-44da-a656-98565482ec3a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521236652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.52124815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806,"session_id":"b3006d94-13ee-44da-a656-98565482ec3a"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:24:48.521260287Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 94 len 28672 data: 11 10 8 8 9 9 9 02/30 Write block 129 len 16384 data: 8 7 4 5 03/30 Read block 565 len 4096 04/30 Read block 480 len 4096 05/30 Read block 43 len 24576 06/30 Read block 445 len 28672 07/30 Read block 480 len 12288 08/30 Flush 09/30 Write block 225 len 20480 data: 6 7 8 10 10 10/30 Read block 333 len 36864 11/30 Write block 336 len 28672 data: 11 9 12 15 11 9 10 12/30 Flush 13/30 Write block 79 len 24576 data: 13 14 13 13 11 14 14/30 Write block 106 len 28672 data: 9 11 16 15 15 12 12 15/30 Write block 216 len 16384 data: 8 8 9 9 16/30 Read block 91 len 4096 17/30 Read block 592 len 20480 18/30 Write block 61 len 16384 data: 9 11 10 10 19/30 Read block 168 len 16384 20/30 Write block 193 len 20480 data: 15 17 14 14 11 21/30 Write block 157 len 4096 data: 15 22/30 Write block 417 len 16384 data: 9 9 10 10 23/30 Write block 496 len 24576 data: 14 12 11 13 13 8 24/30 Read block 368 len 8192 25/30 Read block 405 len 32768 26/30 Write block 198 len 16384 data: 10 9 7 10 27/30 Read block 229 len 16384 28/30 Read block 182 len 36864 29/30 Write block 0 len 16384 data: 3 6 9 9 30/30 Write block 271 len 28672 data: 10 13 11 9 8 11 13 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 6 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:24:48.542465585Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 247 247 247 T T T 1 020-039 148 148 148 247 247 247 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 152 152 152 251 251 251 T T T 4 080-099 154 154 154 253 253 253 T T T 5 100-119 154 154 154 253 253 253 T T T 6 120-139 154 154 154 253 253 253 F F F 7 140-159 153 153 153 252 252 252 T T T 8 160-179 151 151 151 250 250 250 F F F 9 180-199 153 153 153 252 252 252 T T T 10 200-219 147 147 147 246 246 246 T T T 11 220-239 154 154 154 254 254 254 F F F 12 240-259 153 153 153 252 252 252 F F F 13 260-279 151 151 151 250 250 250 T T T 14 280-299 153 153 153 252 252 252 F F F 15 300-319 153 153 153 252 252 252 F F F 16 320-339 154 154 154 254 254 254 F F F 17 340-359 154 154 154 254 254 254 F F F 18 360-379 152 152 152 251 251 251 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 148 148 148 247 247 247 T T T 21 420-439 142 142 142 240 240 240 T T T 22 440-459 148 148 148 247 247 247 F F F 23 460-479 148 148 148 247 247 247 F F F 24 480-499 148 148 148 247 247 247 T T T 25 500-519 151 151 151 250 250 250 T T T 26 520-539 152 152 152 251 251 251 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 153 153 153 252 252 252 F F F 29 580-599 153 153 153 252 252 252 F F F Max gen: 154, Max flush: 254 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 255 255 255 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 155 155 155 255 255 255 F F F 4 080-099 155 155 155 255 255 255 F F F 5 100-119 155 155 155 255 255 255 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 155 155 155 255 255 255 F F F 8 160-179 151 151 151 250 250 250 F F F 9 180-199 155 155 155 255 255 255 F F F 10 200-219 155 155 155 255 255 255 F F F 11 220-239 154 154 154 254 254 254 F F F 12 240-259 153 153 153 252 252 252 F F F 13 260-279 155 155 155 255 255 255 F F F 14 280-299 153 153 153 252 252 252 F F F 15 300-319 153 153 153 252 252 252 F F F 16 320-339 154 154 154 254 254 254 F F F 17 340-359 154 154 154 254 254 254 F F F 18 360-379 152 152 152 251 251 251 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 155 155 155 255 255 255 F F F 21 420-439 155 155 155 255 255 255 F F F 22 440-459 148 148 148 247 247 247 F F F 23 460-479 148 148 148 247 247 247 F F F 24 480-499 155 155 155 255 255 255 F F F 25 500-519 155 155 155 255 255 255 F F F 26 520-539 152 152 152 251 251 251 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 153 153 153 252 252 252 F F F 29 580-599 153 153 153 252 252 252 F F F Max gen: 155, Max flush: 255 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.838059857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.884278997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.885322814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.885378174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.885396621Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.885421326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814} {"msg":"Crucible 60b4de16-ef91-457a-a08d-23e836761785 has session id: 361f42d3-47f5-422a-aca3-0b626a60d99d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886114978Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Upstairs opts: Upstairs UUID: 60b4de16-ef91-457a-a08d-23e836761785, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886152851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Crucible stats registered with UUID: 60b4de16-ef91-457a-a08d-23e836761785","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886168238Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:02.886180866Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886789113Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886814488Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814} {"msg":"60b4de16-ef91-457a-a08d-23e836761785 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886888335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886906901Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"0","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886939215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"0","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886953672Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"1","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.88696825Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"1","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.886981618Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"2","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887004773Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"2","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887035328Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887061283Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887139738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60930)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887173402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62014)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887198497Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60760)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887225012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.887969965Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"2","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.888019615Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"0","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.888622564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"2","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.888756889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"0","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.888874137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"1","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.88924102Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.8893987Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.889540454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"1","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[0]R flush_numbers[0..12]: [255, 247, 252, 255, 255, 255, 253, 255, 250, 255, 255, 254]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.89010279Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[0]R generation[0..12]: [155, 148, 153, 155, 155, 155, 154, 155, 151, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890129145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890144212Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[1]R flush_numbers[0..12]: [255, 247, 252, 255, 255, 255, 253, 255, 250, 255, 255, 254]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.89015825Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[1]R generation[0..12]: [155, 148, 153, 155, 155, 155, 154, 155, 151, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890171767Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890185385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[2]R flush_numbers[0..12]: [255, 247, 252, 255, 255, 255, 253, 255, 250, 255, 255, 254]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890199012Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[2]R generation[0..12]: [155, 148, 153, 155, 155, 155, 154, 155, 151, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.89021229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890225917Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890238365Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890259401Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890271989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890284436Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"":"downstairs","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890297354Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890309022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"0","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890325599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"1","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890338886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"client":"2","":"downstairs","session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890352224Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890364402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"60b4de16-ef91-457a-a08d-23e836761785 is now active with session: 361f42d3-47f5-422a-aca3-0b626a60d99d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890377029Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890388617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814,"session_id":"361f42d3-47f5-422a-aca3-0b626a60d99d"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:02.890400755Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 12288 data: 15 12 11 02/30 Read block 386 len 24576 03/30 Flush 04/30 Write block 2 len 40960 data: 10 10 10 10 12 12 14 19 18 19 05/30 Read block 29 len 12288 06/30 Write block 334 len 24576 data: 13 12 12 10 13 16 07/30 Flush 08/30 Write block 484 len 40960 data: 18 17 18 16 15 10 12 13 14 12 09/30 Read block 129 len 32768 10/30 Read block 354 len 32768 11/30 Read block 15 len 32768 12/30 Read block 265 len 20480 13/30 Write block 464 len 8192 data: 11 10 14/30 Write block 599 len 4096 data: 2 15/30 Read block 261 len 28672 16/30 Write block 408 len 36864 data: 11 11 12 11 11 11 12 10 10 17/30 Flush 18/30 Write block 493 len 8192 data: 13 13 19/30 Flush 20/30 Write block 569 len 8192 data: 10 11 21/30 Write block 174 len 16384 data: 15 15 15 15 22/30 Write block 588 len 40960 data: 9 11 8 5 5 6 7 7 6 4 23/30 Read block 129 len 20480 24/30 Write block 299 len 24576 data: 7 8 9 7 6 7 25/30 Read block 587 len 4096 26/30 Write block 83 len 12288 data: 12 15 16 27/30 Write block 435 len 4096 data: 10 28/30 Write block 262 len 28672 data: 13 10 8 10 11 11 8 29/30 Read block 200 len 36864 30/30 Read block 486 len 20480 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:02.911553452Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 257 257 257 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 155 155 155 255 255 255 F F F 4 080-099 155 155 155 255 255 255 T T T 5 100-119 155 155 155 255 255 255 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 155 155 155 255 255 255 F F F 8 160-179 151 151 151 250 250 250 T T T 9 180-199 156 156 156 256 256 256 F F F 10 200-219 155 155 155 255 255 255 F F F 11 220-239 154 154 154 254 254 254 F F F 12 240-259 153 153 153 252 252 252 F F F 13 260-279 155 155 155 255 255 255 T T T 14 280-299 153 153 153 252 252 252 T T T 15 300-319 153 153 153 252 252 252 T T T 16 320-339 156 156 156 257 257 257 F F F 17 340-359 154 154 154 254 254 254 F F F 18 360-379 152 152 152 251 251 251 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 156 156 156 258 258 258 F F F 21 420-439 155 155 155 255 255 255 T T T 22 440-459 148 148 148 247 247 247 F F F 23 460-479 156 156 156 258 258 258 F F F 24 480-499 156 156 156 259 259 259 F F F 25 500-519 155 155 155 255 255 255 F F F 26 520-539 152 152 152 251 251 251 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 153 153 153 252 252 252 T T T 29 580-599 156 156 156 258 258 258 T T T Max gen: 156, Max flush: 259 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 257 257 257 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 155 155 155 255 255 255 F F F 4 080-099 157 157 157 260 260 260 F F F 5 100-119 155 155 155 255 255 255 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 155 155 155 255 255 255 F F F 8 160-179 157 157 157 260 260 260 F F F 9 180-199 156 156 156 256 256 256 F F F 10 200-219 155 155 155 255 255 255 F F F 11 220-239 154 154 154 254 254 254 F F F 12 240-259 153 153 153 252 252 252 F F F 13 260-279 157 157 157 260 260 260 F F F 14 280-299 157 157 157 260 260 260 F F F 15 300-319 157 157 157 260 260 260 F F F 16 320-339 156 156 156 257 257 257 F F F 17 340-359 154 154 154 254 254 254 F F F 18 360-379 152 152 152 251 251 251 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 156 156 156 258 258 258 F F F 21 420-439 157 157 157 260 260 260 F F F 22 440-459 148 148 148 247 247 247 F F F 23 460-479 156 156 156 258 258 258 F F F 24 480-499 156 156 156 259 259 259 F F F 25 500-519 155 155 155 255 255 255 F F F 26 520-539 152 152 152 251 251 251 F F F 27 540-559 148 148 148 247 247 247 F F F 28 560-579 157 157 157 260 260 260 F F F 29 580-599 157 157 157 260 260 260 F F F Max gen: 157, Max flush: 260 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.188476058Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.233565518Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.234272057Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.234316369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.234331746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.234346423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822} {"msg":"Crucible e55ae7c7-5bb9-4aee-8c80-9a0e38d36c6b has session id: 56a74117-05d4-48ca-b43b-0aec770f8129","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.234990814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Upstairs opts: Upstairs UUID: e55ae7c7-5bb9-4aee-8c80-9a0e38d36c6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235032106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Crucible stats registered with UUID: e55ae7c7-5bb9-4aee-8c80-9a0e38d36c6b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235055402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:07.235075458Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235637474Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235664949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822} {"msg":"e55ae7c7-5bb9-4aee-8c80-9a0e38d36c6b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235746704Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235765211Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"0","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235798405Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"0","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235813552Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"1","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235827999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"1","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235841166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"2","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235856554Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"2","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235880779Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235907874Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235936139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"ds_connection connected from Ok(127.0.0.1:55608)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.235961614Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"ds_connection connected from Ok(127.0.0.1:65346)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.236021973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"ds_connection connected from Ok(127.0.0.1:45112)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.236138522Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.236669973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"2","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.236820795Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"0","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.237261054Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"2","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.237368584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"0","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.237853694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.237973962Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.238449864Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"1","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.238987435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"1","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[0]R flush_numbers[0..12]: [257, 247, 252, 255, 260, 255, 253, 255, 260, 256, 255, 254]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239504369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[0]R generation[0..12]: [156, 148, 153, 155, 157, 155, 154, 155, 157, 156, 155, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239529684Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239545041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[1]R flush_numbers[0..12]: [257, 247, 252, 255, 260, 255, 253, 255, 260, 256, 255, 254]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239559139Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[1]R generation[0..12]: [156, 148, 153, 155, 157, 155, 154, 155, 157, 156, 155, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239572856Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239586804Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[2]R flush_numbers[0..12]: [257, 247, 252, 255, 260, 255, 253, 255, 260, 256, 255, 254]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239600521Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[2]R generation[0..12]: [156, 148, 153, 155, 157, 155, 154, 155, 157, 156, 155, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239613989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239627786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:25:07.239644403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.23966045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239673318Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239686265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"":"downstairs","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239699333Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239710971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"0","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239724278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"1","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239737346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"client":"2","":"downstairs","session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239750383Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239778248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"e55ae7c7-5bb9-4aee-8c80-9a0e38d36c6b is now active with session: 56a74117-05d4-48ca-b43b-0aec770f8129","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239792116Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239803904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822,"session_id":"56a74117-05d4-48ca-b43b-0aec770f8129"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:07.239815841Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 222 len 4096 data: 8 02/30 Read block 508 len 28672 03/30 Read block 341 len 36864 04/30 Write block 175 len 28672 data: 16 16 16 14 17 17 16 05/30 Write block 427 len 12288 data: 9 8 10 06/30 Write block 448 len 12288 data: 7 10 10 07/30 Read block 77 len 32768 08/30 Flush 09/30 Write block 111 len 4096 data: 13 10/30 Write block 319 len 32768 data: 19 19 14 12 12 11 11 12 11/30 Write block 469 len 12288 data: 12 9 10 12/30 Write block 553 len 20480 data: 10 9 9 9 10 13/30 Read block 434 len 36864 14/30 Write block 81 len 8192 data: 14 14 15/30 Write block 356 len 36864 data: 17 18 19 20 18 17 13 12 12 16/30 Read block 259 len 32768 17/30 Read block 118 len 8192 18/30 Write block 526 len 32768 data: 10 9 9 8 9 10 9 7 19/30 Flush 20/30 Write block 142 len 8192 data: 12 8 21/30 Read block 295 len 12288 22/30 Read block 181 len 40960 23/30 Write block 228 len 12288 data: 11 11 12 24/30 Write block 485 len 36864 data: 18 19 17 16 11 13 14 15 14 25/30 Read block 234 len 24576 26/30 Write block 228 len 36864 data: 12 12 13 12 13 15 10 11 12 27/30 Read block 466 len 28672 28/30 Write block 412 len 12288 data: 12 12 13 29/30 Read block 326 len 32768 30/30 Read block 106 len 32768 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:07.260112687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 257 257 257 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 155 155 155 255 255 255 F F F 4 080-099 158 158 158 262 262 262 F F F 5 100-119 158 158 158 262 262 262 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 155 155 155 255 255 255 T T T 8 160-179 158 158 158 261 261 261 F F F 9 180-199 158 158 158 261 261 261 F F F 10 200-219 155 155 155 255 255 255 F F F 11 220-239 158 158 158 261 261 261 T T T 12 240-259 153 153 153 252 252 252 F F F 13 260-279 157 157 157 260 260 260 F F F 14 280-299 157 157 157 260 260 260 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 158 158 158 262 262 262 F F F 17 340-359 158 158 158 262 262 262 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 156 156 156 258 258 258 T T T 21 420-439 158 158 158 261 261 261 F F F 22 440-459 158 158 158 261 261 261 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 156 156 156 259 259 259 T T T 25 500-519 155 155 155 255 255 255 F F F 26 520-539 158 158 158 262 262 262 F F F 27 540-559 158 158 158 262 262 262 F F F 28 560-579 157 157 157 260 260 260 F F F 29 580-599 157 157 157 260 260 260 F F F Max gen: 158, Max flush: 262 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 257 257 257 F F F 1 020-039 148 148 148 247 247 247 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 155 155 155 255 255 255 F F F 4 080-099 158 158 158 262 262 262 F F F 5 100-119 158 158 158 262 262 262 F F F 6 120-139 154 154 154 253 253 253 F F F 7 140-159 159 159 159 263 263 263 F F F 8 160-179 158 158 158 261 261 261 F F F 9 180-199 158 158 158 261 261 261 F F F 10 200-219 155 155 155 255 255 255 F F F 11 220-239 159 159 159 263 263 263 F F F 12 240-259 153 153 153 252 252 252 F F F 13 260-279 157 157 157 260 260 260 F F F 14 280-299 157 157 157 260 260 260 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 158 158 158 262 262 262 F F F 17 340-359 158 158 158 262 262 262 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 153 153 153 252 252 252 F F F 20 400-419 159 159 159 263 263 263 F F F 21 420-439 158 158 158 261 261 261 F F F 22 440-459 158 158 158 261 261 261 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 159 159 159 263 263 263 F F F 25 500-519 155 155 155 255 255 255 F F F 26 520-539 158 158 158 262 262 262 F F F 27 540-559 158 158 158 262 262 262 F F F 28 560-579 157 157 157 260 260 260 F F F 29 580-599 157 157 157 260 260 260 F F F Max gen: 159, Max flush: 263 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.507793882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.552902708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.553625684Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.553669746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.553684723Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.553699561Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830} {"msg":"Crucible 34f8092d-3a59-4929-94d7-7c33f417f6dd has session id: f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.554380994Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Upstairs opts: Upstairs UUID: 34f8092d-3a59-4929-94d7-7c33f417f6dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.554417068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Crucible stats registered with UUID: 34f8092d-3a59-4929-94d7-7c33f417f6dd","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.554432435Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:11.554444893Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.554874253Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.554916326Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830} {"msg":"34f8092d-3a59-4929-94d7-7c33f417f6dd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555019816Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555040233Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"0","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555067138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"0","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555083425Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"1","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.55510963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"1","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555136315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"2","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.55516548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"2","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555291906Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555334728Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555367422Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52361)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555402436Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40682)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.55554053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47902)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.555567235Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.556007374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"0","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.556059804Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"2","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.556627849Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"2","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.556735819Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"0","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.557177617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.557280308Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.557892165Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"1","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.558410879Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"1","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[0]R flush_numbers[0..12]: [257, 247, 252, 255, 262, 262, 253, 263, 261, 261, 255, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.558917486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[0]R generation[0..12]: [156, 148, 153, 155, 158, 158, 154, 159, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.558937422Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.558951719Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[1]R flush_numbers[0..12]: [257, 247, 252, 255, 262, 262, 253, 263, 261, 261, 255, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.558969266Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[1]R generation[0..12]: [156, 148, 153, 155, 158, 158, 154, 159, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.558983173Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.558997111Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[2]R flush_numbers[0..12]: [257, 247, 252, 255, 262, 262, 253, 263, 261, 261, 255, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559010628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[2]R generation[0..12]: [156, 148, 153, 155, 158, 158, 154, 159, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559024006Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559037433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559049871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"Wait for a query_work_queue command to finish before sending IO msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559068807Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559082355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559094973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"":"downstairs","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.5591077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559119488Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"0","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559132516Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"1","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559145353Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"client":"2","":"downstairs","session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559157981Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559172278Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"34f8092d-3a59-4929-94d7-7c33f417f6dd is now active with session: f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559184886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559196414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830,"session_id":"f39a34bc-beb1-4bb2-a3d6-0a98bcde6aa1"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:11.559208262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 542 len 20480 data: 14 14 15 16 16 02/30 Read block 208 len 12288 03/30 Write block 106 len 32768 data: 10 12 17 16 16 14 13 15 04/30 Write block 191 len 40960 data: 8 14 16 18 15 16 13 12 10 8 05/30 Write block 566 len 8192 data: 15 14 06/30 Read block 452 len 4096 07/30 Write block 571 len 36864 data: 8 10 12 13 13 11 13 15 17 08/30 Write block 582 len 4096 data: 18 09/30 Read block 387 len 40960 10/30 Read block 413 len 28672 11/30 Read block 477 len 12288 12/30 Flush 13/30 Write block 242 len 20480 data: 9 11 8 8 11 14/30 Read block 22 len 12288 15/30 Write block 197 len 4096 data: 14 16/30 Write block 398 len 28672 data: 9 13 14 14 12 12 12 17/30 Read block 557 len 36864 18/30 Read block 500 len 28672 19/30 Read block 224 len 4096 20/30 Write block 170 len 4096 data: 11 21/30 Write block 555 len 24576 data: 10 10 11 9 11 12 22/30 Read block 199 len 36864 23/30 Write block 34 len 4096 data: 8 24/30 Read block 473 len 32768 25/30 Read block 401 len 16384 26/30 Write block 123 len 28672 data: 11 12 10 10 12 10 9 27/30 Read block 574 len 36864 28/30 Read block 147 len 16384 29/30 Read block 265 len 28672 30/30 Read block 199 len 24576 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:11.579939657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 257 257 257 F F F 1 020-039 148 148 148 247 247 247 T T T 2 040-059 153 153 153 252 252 252 F F F 3 060-079 155 155 155 255 255 255 F F F 4 080-099 158 158 158 262 262 262 F F F 5 100-119 160 160 160 264 264 264 F F F 6 120-139 154 154 154 253 253 253 T T T 7 140-159 159 159 159 263 263 263 F F F 8 160-179 158 158 158 261 261 261 T T T 9 180-199 160 160 160 264 264 264 T T T 10 200-219 160 160 160 264 264 264 F F F 11 220-239 159 159 159 263 263 263 F F F 12 240-259 153 153 153 252 252 252 T T T 13 260-279 157 157 157 260 260 260 F F F 14 280-299 157 157 157 260 260 260 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 158 158 158 262 262 262 F F F 17 340-359 158 158 158 262 262 262 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 153 153 153 252 252 252 T T T 20 400-419 159 159 159 263 263 263 T T T 21 420-439 158 158 158 261 261 261 F F F 22 440-459 158 158 158 261 261 261 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 159 159 159 263 263 263 F F F 25 500-519 155 155 155 255 255 255 F F F 26 520-539 158 158 158 262 262 262 F F F 27 540-559 160 160 160 264 264 264 T T T 28 560-579 160 160 160 264 264 264 T T T 29 580-599 160 160 160 264 264 264 F F F Max gen: 160, Max flush: 264 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 257 257 257 F F F 1 020-039 161 161 161 265 265 265 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 155 155 155 255 255 255 F F F 4 080-099 158 158 158 262 262 262 F F F 5 100-119 160 160 160 264 264 264 F F F 6 120-139 161 161 161 265 265 265 F F F 7 140-159 159 159 159 263 263 263 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 161 161 161 265 265 265 F F F 10 200-219 160 160 160 264 264 264 F F F 11 220-239 159 159 159 263 263 263 F F F 12 240-259 161 161 161 265 265 265 F F F 13 260-279 157 157 157 260 260 260 F F F 14 280-299 157 157 157 260 260 260 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 158 158 158 262 262 262 F F F 17 340-359 158 158 158 262 262 262 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 161 161 161 265 265 265 F F F 20 400-419 161 161 161 265 265 265 F F F 21 420-439 158 158 158 261 261 261 F F F 22 440-459 158 158 158 261 261 261 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 159 159 159 263 263 263 F F F 25 500-519 155 155 155 255 255 255 F F F 26 520-539 158 158 158 262 262 262 F F F 27 540-559 161 161 161 265 265 265 F F F 28 560-579 161 161 161 265 265 265 F F F 29 580-599 160 160 160 264 264 264 F F F Max gen: 161, Max flush: 265 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.850482973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:15.894595643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.895031572Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.92698988Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.927652548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.92769574Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.927712427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.927730094Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838} {"msg":"Crucible 4a917ca3-07ce-40e6-b0c4-4bceb1a66697 has session id: 8ea7c958-9720-48ab-a46b-816ce60b6e52","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.928366036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Upstairs opts: Upstairs UUID: 4a917ca3-07ce-40e6-b0c4-4bceb1a66697, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.928407788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Crucible stats registered with UUID: 4a917ca3-07ce-40e6-b0c4-4bceb1a66697","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.928427674Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:15.928441222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.928886889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.928920953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838} {"msg":"4a917ca3-07ce-40e6-b0c4-4bceb1a66697 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.928973153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929003968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"0","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929032862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"0","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.92904694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"1","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929061327Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"1","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929074605Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"2","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929134414Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"2","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929166008Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929183085Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929197212Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:37164)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929213199Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:51035)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.929237724Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:51653)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.92926255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.930051534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"1","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.930079599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"2","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.930094776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"0","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.930663571Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"1","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.93077652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"0","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.930804415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"2","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931341915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931461843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[0]R flush_numbers[0..12]: [257, 265, 252, 255, 262, 264, 265, 263, 265, 265, 264, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931527221Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[0]R generation[0..12]: [156, 161, 153, 155, 158, 160, 161, 159, 161, 161, 160, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931554936Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931570063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[1]R flush_numbers[0..12]: [257, 265, 252, 255, 262, 264, 265, 263, 265, 265, 264, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.93158389Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[1]R generation[0..12]: [156, 161, 153, 155, 158, 160, 161, 159, 161, 161, 160, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931597798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931611736Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[2]R flush_numbers[0..12]: [257, 265, 252, 255, 262, 264, 265, 263, 265, 265, 264, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931625413Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[2]R generation[0..12]: [156, 161, 153, 155, 158, 160, 161, 159, 161, 161, 160, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931638751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931655977Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931669325Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931689811Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931702509Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931715196Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"":"downstairs","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931728064Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931739622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"0","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931752479Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"1","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931765157Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"client":"2","":"downstairs","session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931777825Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931789783Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"4a917ca3-07ce-40e6-b0c4-4bceb1a66697 is now active with session: 8ea7c958-9720-48ab-a46b-816ce60b6e52","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.9318021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931813798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838,"session_id":"8ea7c958-9720-48ab-a46b-816ce60b6e52"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:15.931825916Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 317 len 28672 data: 16 18 20 20 15 13 13 02/30 Write block 589 len 32768 data: 12 9 6 6 7 8 8 7 03/30 Write block 103 len 32768 data: 9 8 9 11 13 18 17 17 04/30 Read block 464 len 16384 05/30 Write block 485 len 32768 data: 19 20 18 17 12 14 15 16 06/30 Read block 3 len 8192 07/30 Read block 310 len 20480 08/30 Write block 436 len 20480 data: 12 12 12 12 14 09/30 Write block 345 len 40960 data: 14 14 14 13 12 11 9 9 12 12 10/30 Write block 536 len 12288 data: 8 12 11 11/30 Flush 12/30 Write block 246 len 36864 data: 12 12 13 13 14 15 15 15 17 13/30 Read block 410 len 24576 14/30 Read block 231 len 20480 15/30 Write block 84 len 40960 data: 16 17 11 14 12 11 9 8 9 11 16/30 Write block 494 len 36864 data: 14 14 15 13 12 14 14 9 10 17/30 Write block 12 len 36864 data: 16 17 15 16 13 14 15 14 15 18/30 Write block 340 len 32768 data: 12 10 11 11 17 15 15 15 19/30 Write block 339 len 8192 data: 17 13 20/30 Read block 540 len 4096 21/30 Read block 269 len 16384 22/30 Write block 76 len 16384 data: 14 11 11 14 23/30 Write block 380 len 4096 data: 11 24/30 Write block 280 len 8192 data: 13 10 25/30 Write block 71 len 4096 data: 21 26/30 Flush 27/30 Write block 129 len 28672 data: 10 8 5 6 8 5 9 28/30 Read block 483 len 12288 29/30 Write block 302 len 40960 data: 8 7 8 10 10 11 15 13 16 17 30/30 Write block 93 len 20480 data: 12 12 11 9 9 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:15.951015576Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 267 267 267 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 162 162 162 267 267 267 T T T 5 100-119 162 162 162 266 266 266 F F F 6 120-139 161 161 161 265 265 265 T T T 7 140-159 159 159 159 263 263 263 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 161 161 161 265 265 265 F F F 10 200-219 160 160 160 264 264 264 F F F 11 220-239 159 159 159 263 263 263 F F F 12 240-259 162 162 162 267 267 267 F F F 13 260-279 157 157 157 260 260 260 F F F 14 280-299 162 162 162 267 267 267 F F F 15 300-319 162 162 162 266 266 266 T T T 16 320-339 162 162 162 267 267 267 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 162 162 162 267 267 267 F F F 20 400-419 161 161 161 265 265 265 F F F 21 420-439 162 162 162 266 266 266 F F F 22 440-459 162 162 162 266 266 266 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 162 162 162 267 267 267 F F F 26 520-539 162 162 162 266 266 266 F F F 27 540-559 161 161 161 265 265 265 F F F 28 560-579 161 161 161 265 265 265 F F F 29 580-599 162 162 162 266 266 266 F F F Max gen: 162, Max flush: 267 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 267 267 267 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 163 163 163 268 268 268 F F F 5 100-119 162 162 162 266 266 266 F F F 6 120-139 163 163 163 268 268 268 F F F 7 140-159 159 159 159 263 263 263 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 161 161 161 265 265 265 F F F 10 200-219 160 160 160 264 264 264 F F F 11 220-239 159 159 159 263 263 263 F F F 12 240-259 162 162 162 267 267 267 F F F 13 260-279 157 157 157 260 260 260 F F F 14 280-299 162 162 162 267 267 267 F F F 15 300-319 163 163 163 268 268 268 F F F 16 320-339 162 162 162 267 267 267 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 162 162 162 267 267 267 F F F 20 400-419 161 161 161 265 265 265 F F F 21 420-439 162 162 162 266 266 266 F F F 22 440-459 162 162 162 266 266 266 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 162 162 162 267 267 267 F F F 26 520-539 162 162 162 266 266 266 F F F 27 540-559 161 161 161 265 265 265 F F F 28 560-579 161 161 161 265 265 265 F F F 29 580-599 162 162 162 266 266 266 F F F Max gen: 163, Max flush: 268 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.271086851Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:20.327184654Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.328465337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.365186444Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.367189364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.367246013Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.36726396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.367282657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846} {"msg":"Crucible 9a650f0f-52a7-498d-a13e-fedd191891e2 has session id: bff3ae1e-5632-4ffd-90a4-3094ba06d4d8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.368087717Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Upstairs opts: Upstairs UUID: 9a650f0f-52a7-498d-a13e-fedd191891e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.368162584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Crucible stats registered with UUID: 9a650f0f-52a7-498d-a13e-fedd191891e2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.368186259Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:20.368202696Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.368870133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.368900717Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846} {"msg":"9a650f0f-52a7-498d-a13e-fedd191891e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.368967355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.368998319Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"0","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369035512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"0","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.36904989Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"1","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369064457Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"1","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369079144Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"2","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369093462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"2","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369112738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369140903Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369167208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:46892)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369412952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41262)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369477411Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61154)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.369505715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.370665791Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"1","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.370699254Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"2","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.371087993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"0","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.371662616Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"1","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.371707758Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"2","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372056134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"0","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372467667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372499532Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[0]R flush_numbers[0..12]: [267, 267, 252, 267, 268, 266, 268, 263, 265, 265, 264, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372788538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[0]R generation[0..12]: [162, 162, 153, 162, 163, 162, 163, 159, 161, 161, 160, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372818293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372842178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[1]R flush_numbers[0..12]: [267, 267, 252, 267, 268, 266, 268, 263, 265, 265, 264, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372857245Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[1]R generation[0..12]: [162, 162, 153, 162, 163, 162, 163, 159, 161, 161, 160, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372871273Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.37288534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[2]R flush_numbers[0..12]: [267, 267, 252, 267, 268, 266, 268, 263, 265, 265, 264, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372899087Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[2]R generation[0..12]: [162, 162, 153, 162, 163, 162, 163, 159, 161, 161, 160, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372912345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372926182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.37293898Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372952118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372965345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.372978363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"":"downstairs","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:25:20.372991481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.373015256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"0","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.373037782Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"1","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.373061428Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"client":"2","":"downstairs","session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.373077005Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.373089662Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"9a650f0f-52a7-498d-a13e-fedd191891e2 is now active with session: bff3ae1e-5632-4ffd-90a4-3094ba06d4d8","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.37310276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.373114798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846,"session_id":"bff3ae1e-5632-4ffd-90a4-3094ba06d4d8"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:20.373127225Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 12288 data: 12 12 9 02/30 Read block 22 len 36864 03/30 Write block 395 len 28672 data: 8 7 7 10 14 15 15 04/30 Write block 251 len 16384 data: 16 16 16 18 05/30 Write block 137 len 24576 data: 12 12 14 11 9 13 06/30 Write block 331 len 24576 data: 7 12 13 14 13 13 07/30 Read block 419 len 24576 08/30 Flush 09/30 Write block 281 len 36864 data: 11 11 11 12 9 8 7 9 12 10/30 Read block 585 len 28672 11/30 Write block 270 len 20480 data: 10 11 14 12 10 12/30 Flush 13/30 Flush 14/30 Write block 537 len 32768 data: 13 12 12 12 11 15 15 16 15/30 Read block 238 len 32768 16/30 Read block 90 len 36864 17/30 Write block 149 len 36864 data: 13 12 13 14 14 15 15 14 16 18/30 Flush 19/30 Write block 396 len 20480 data: 8 8 11 15 16 20/30 Write block 62 len 28672 data: 12 11 11 15 12 10 12 21/30 Write block 136 len 28672 data: 10 13 13 15 12 10 14 22/30 Read block 492 len 32768 23/30 Read block 203 len 16384 24/30 Write block 536 len 16384 data: 9 14 13 13 25/30 Read block 327 len 4096 26/30 Flush 27/30 Write block 318 len 4096 data: 19 28/30 Write block 197 len 32768 data: 15 13 11 9 11 13 14 14 29/30 Read block 344 len 28672 30/30 Read block 6 len 32768 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:20.397219879Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 267 267 267 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 164 164 164 273 273 273 F F F 4 080-099 163 163 163 268 268 268 F F F 5 100-119 162 162 162 266 266 266 F F F 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 161 161 161 265 265 265 T T T 10 200-219 160 160 160 264 264 264 T T T 11 220-239 159 159 159 263 263 263 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 164 164 164 270 270 270 F F F 14 280-299 164 164 164 270 270 270 F F F 15 300-319 163 163 163 268 268 268 T T T 16 320-339 164 164 164 269 269 269 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 164 164 164 273 273 273 F F F 20 400-419 164 164 164 273 273 273 F F F 21 420-439 162 162 162 266 266 266 F F F 22 440-459 162 162 162 266 266 266 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 162 162 162 267 267 267 F F F 26 520-539 164 164 164 273 273 273 F F F 27 540-559 164 164 164 272 272 272 F F F 28 560-579 161 161 161 265 265 265 F F F 29 580-599 162 162 162 266 266 266 F F F Max gen: 164, Max flush: 273 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 267 267 267 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 164 164 164 273 273 273 F F F 4 080-099 163 163 163 268 268 268 F F F 5 100-119 162 162 162 266 266 266 F F F 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 165 165 165 274 274 274 F F F 10 200-219 165 165 165 274 274 274 F F F 11 220-239 159 159 159 263 263 263 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 164 164 164 270 270 270 F F F 14 280-299 164 164 164 270 270 270 F F F 15 300-319 165 165 165 274 274 274 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 158 158 158 262 262 262 F F F 19 380-399 164 164 164 273 273 273 F F F 20 400-419 164 164 164 273 273 273 F F F 21 420-439 162 162 162 266 266 266 F F F 22 440-459 162 162 162 266 266 266 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 162 162 162 267 267 267 F F F 26 520-539 164 164 164 273 273 273 F F F 27 540-559 164 164 164 272 272 272 F F F 28 560-579 161 161 161 265 265 265 F F F 29 580-599 162 162 162 266 266 266 F F F Max gen: 165, Max flush: 274 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.72227963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.767370429Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.76806798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.768108763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.76812348Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.768137827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854} {"msg":"Crucible 82be9cf2-462c-4b06-bcaf-4a331a14c659 has session id: 6a113f0a-de15-458c-954a-d3396a356645","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.768802115Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Upstairs opts: Upstairs UUID: 82be9cf2-462c-4b06-bcaf-4a331a14c659, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.768836498Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Crucible stats registered with UUID: 82be9cf2-462c-4b06-bcaf-4a331a14c659","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.768861134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:24.76888212Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769343694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769371799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854} {"msg":"82be9cf2-462c-4b06-bcaf-4a331a14c659 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769433077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769457943Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"0","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769483558Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"0","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769497455Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"1","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769517862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"1","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769531359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"2","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769545086Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"2","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769560314Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.76958363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769604276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"ds_connection connected from Ok(127.0.0.1:52741)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769627271Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"ds_connection connected from Ok(127.0.0.1:32786)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769650667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"ds_connection connected from Ok(127.0.0.1:54356)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.769882834Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.770389181Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"2","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.770427923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"0","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.771115786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"0","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.77114961Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"2","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.771531859Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"1","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.771727843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.771816596Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772075638Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"1","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[0]R flush_numbers[0..12]: [267, 267, 252, 273, 268, 266, 273, 273, 265, 274, 274, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772587134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[0]R generation[0..12]: [162, 162, 153, 164, 163, 162, 164, 164, 161, 165, 165, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.77260736Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772622417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[1]R flush_numbers[0..12]: [267, 267, 252, 273, 268, 266, 273, 273, 265, 274, 274, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772636595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[1]R generation[0..12]: [162, 162, 153, 164, 163, 162, 164, 164, 161, 165, 165, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772650112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.77266384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[2]R flush_numbers[0..12]: [267, 267, 252, 273, 268, 266, 273, 273, 265, 274, 274, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772677317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[2]R generation[0..12]: [162, 162, 153, 164, 163, 162, 164, 164, 161, 165, 165, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772690525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772703862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.77271628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772729057Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772745274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.77276763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"":"downstairs","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772780608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772792286Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"0","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772805214Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"1","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772818041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"client":"2","":"downstairs","session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772830729Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772842567Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"82be9cf2-462c-4b06-bcaf-4a331a14c659 is now active with session: 6a113f0a-de15-458c-954a-d3396a356645","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772854974Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.772866502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854,"session_id":"6a113f0a-de15-458c-954a-d3396a356645"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:24.77287834Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 8 len 8192 data: 15 20 02/30 Write block 366 len 20480 data: 13 16 15 16 14 03/30 Write block 7 len 20480 data: 13 16 21 19 20 04/30 Flush 05/30 Write block 439 len 24576 data: 13 15 13 13 10 10 06/30 Write block 453 len 20480 data: 12 11 12 12 13 07/30 Read block 198 len 12288 08/30 Read block 435 len 8192 09/30 Write block 406 len 28672 data: 15 14 12 12 13 12 13 10/30 Read block 194 len 16384 11/30 Write block 377 len 28672 data: 14 14 12 12 10 11 12 12/30 Flush 13/30 Write block 558 len 20480 data: 10 12 13 15 14 14/30 Read block 385 len 28672 15/30 Write block 283 len 4096 data: 12 16/30 Read block 552 len 12288 17/30 Read block 323 len 28672 18/30 Write block 109 len 12288 data: 18 18 15 19/30 Write block 514 len 16384 data: 10 10 12 11 20/30 Read block 338 len 32768 21/30 Write block 99 len 36864 data: 10 10 9 10 10 9 10 12 14 22/30 Read block 352 len 36864 23/30 Read block 228 len 32768 24/30 Write block 213 len 16384 data: 10 9 6 9 25/30 Read block 294 len 12288 26/30 Read block 451 len 36864 27/30 Read block 233 len 20480 28/30 Write block 573 len 40960 data: 13 14 14 12 14 16 18 20 19 19 29/30 Read block 231 len 24576 30/30 Write block 283 len 40960 data: 13 13 10 9 8 10 13 14 15 10 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:24.793162648Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 275 275 275 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 164 164 164 273 273 273 F F F 4 080-099 163 163 163 268 268 268 T T T 5 100-119 162 162 162 266 266 266 T T T 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 165 165 165 274 274 274 F F F 10 200-219 165 165 165 274 274 274 T T T 11 220-239 159 159 159 263 263 263 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 164 164 164 270 270 270 F F F 14 280-299 164 164 164 270 270 270 T T T 15 300-319 165 165 165 274 274 274 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 166 166 166 276 276 276 F F F 19 380-399 166 166 166 276 276 276 F F F 20 400-419 166 166 166 276 276 276 F F F 21 420-439 166 166 166 276 276 276 F F F 22 440-459 166 166 166 276 276 276 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 162 162 162 267 267 267 T T T 26 520-539 164 164 164 273 273 273 F F F 27 540-559 164 164 164 272 272 272 T T T 28 560-579 161 161 161 265 265 265 T T T 29 580-599 162 162 162 266 266 266 T T T Max gen: 166, Max flush: 276 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 275 275 275 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 153 153 153 252 252 252 F F F 3 060-079 164 164 164 273 273 273 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 167 167 167 277 277 277 F F F 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 165 165 165 274 274 274 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 159 159 159 263 263 263 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 164 164 164 270 270 270 F F F 14 280-299 167 167 167 277 277 277 F F F 15 300-319 165 165 165 274 274 274 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 166 166 166 276 276 276 F F F 19 380-399 166 166 166 276 276 276 F F F 20 400-419 166 166 166 276 276 276 F F F 21 420-439 166 166 166 276 276 276 F F F 22 440-459 166 166 166 276 276 276 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 167 167 167 277 277 277 F F F 26 520-539 164 164 164 273 273 273 F F F 27 540-559 167 167 167 277 277 277 F F F 28 560-579 167 167 167 277 277 277 F F F 29 580-599 167 167 167 277 277 277 F F F Max gen: 167, Max flush: 277 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.065903958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:29.1107637Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.111473479Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.144158842Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.145299291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.145344033Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.14536182Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.145379296Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862} {"msg":"Crucible 3f052762-e10f-461c-898c-f20583cbc6d3 has session id: 3b6794c2-9cb9-4d6e-b211-708d8053432f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146015349Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Upstairs opts: Upstairs UUID: 3f052762-e10f-461c-898c-f20583cbc6d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146057751Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Crucible stats registered with UUID: 3f052762-e10f-461c-898c-f20583cbc6d3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146076138Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:29.146091395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146575795Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.14660242Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862} {"msg":"3f052762-e10f-461c-898c-f20583cbc6d3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146671357Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146689744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"0","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146718969Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"0","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146742774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"1","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146770869Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"1","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146797304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"2","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146825879Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"2","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146853564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146881159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146910023Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55519)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.146947217Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46062)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.14698093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34073)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.147011434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.147882544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"2","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.147923686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"1","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.148068469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"0","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.148564657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"1","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.148596101Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"2","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.148674667Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"0","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149213847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.14930821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[0]R flush_numbers[0..12]: [275, 267, 252, 273, 277, 277, 273, 273, 265, 274, 277, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149383396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[0]R generation[0..12]: [166, 162, 153, 164, 167, 167, 164, 164, 161, 165, 167, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149403652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149418369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[1]R flush_numbers[0..12]: [275, 267, 252, 273, 277, 277, 273, 273, 265, 274, 277, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149437026Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[1]R generation[0..12]: [166, 162, 153, 164, 167, 167, 164, 164, 161, 165, 167, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149450983Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149464571Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[2]R flush_numbers[0..12]: [275, 267, 252, 273, 277, 277, 273, 273, 265, 274, 277, 263]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149478108Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[2]R generation[0..12]: [166, 162, 153, 164, 167, 167, 164, 164, 161, 165, 167, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149491396Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149504873Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149517461Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149530199Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:25:29.149542656Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149559713Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"":"downstairs","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149572571Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149584129Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"0","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149597066Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"1","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149609624Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"client":"2","":"downstairs","session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149624661Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149636749Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"3f052762-e10f-461c-898c-f20583cbc6d3 is now active with session: 3b6794c2-9cb9-4d6e-b211-708d8053432f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149649187Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149660924Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862,"session_id":"3b6794c2-9cb9-4d6e-b211-708d8053432f"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:29.149672972Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 380 len 16384 data: 13 11 12 13 02/30 Write block 350 len 8192 data: 12 10 03/30 Read block 150 len 12288 04/30 Write block 311 len 24576 data: 18 19 20 18 15 17 05/30 Read block 434 len 24576 06/30 Read block 13 len 16384 07/30 Read block 398 len 8192 08/30 Flush 09/30 Write block 574 len 32768 data: 15 15 13 15 17 19 21 20 10/30 Flush 11/30 Write block 286 len 40960 data: 10 9 11 14 15 16 11 12 9 11 12/30 Write block 224 len 28672 data: 7 7 8 9 13 13 14 13/30 Write block 400 len 4096 data: 17 14/30 Read block 527 len 8192 15/30 Read block 550 len 4096 16/30 Read block 354 len 36864 17/30 Write block 455 len 12288 data: 13 13 14 18/30 Write block 64 len 8192 data: 12 16 19/30 Write block 340 len 32768 data: 14 11 12 12 18 16 16 16 20/30 Write block 411 len 24576 data: 13 14 13 14 11 11 21/30 Write block 555 len 28672 data: 11 11 12 11 13 14 16 22/30 Flush 23/30 Write block 493 len 36864 data: 15 15 15 16 14 13 15 15 10 24/30 Flush 25/30 Write block 44 len 24576 data: 10 11 11 9 10 14 26/30 Read block 86 len 36864 27/30 Read block 485 len 32768 28/30 Write block 268 len 24576 data: 10 10 11 12 15 13 29/30 Write block 508 len 32768 data: 9 9 9 10 9 10 11 11 30/30 Read block 527 len 8192 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:29.170906744Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 275 275 275 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 153 153 153 252 252 252 T T T 3 060-079 168 168 168 280 280 280 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 167 167 167 277 277 277 F F F 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 165 165 165 274 274 274 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 168 168 168 280 280 280 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 164 164 164 270 270 270 T T T 14 280-299 168 168 168 280 280 280 F F F 15 300-319 168 168 168 278 278 278 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 168 168 168 280 280 280 F F F 18 360-379 166 166 166 276 276 276 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 168 168 168 280 280 280 F F F 21 420-439 166 166 166 276 276 276 F F F 22 440-459 168 168 168 280 280 280 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 168 168 168 281 281 281 F F F 25 500-519 168 168 168 281 281 281 T T T 26 520-539 164 164 164 273 273 273 F F F 27 540-559 168 168 168 280 280 280 F F F 28 560-579 168 168 168 280 280 280 F F F 29 580-599 168 168 168 279 279 279 F F F Max gen: 168, Max flush: 281 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 275 275 275 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 169 169 169 282 282 282 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 167 167 167 277 277 277 F F F 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 161 161 161 265 265 265 F F F 9 180-199 165 165 165 274 274 274 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 168 168 168 280 280 280 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 169 169 169 282 282 282 F F F 14 280-299 168 168 168 280 280 280 F F F 15 300-319 168 168 168 278 278 278 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 168 168 168 280 280 280 F F F 18 360-379 166 166 166 276 276 276 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 168 168 168 280 280 280 F F F 21 420-439 166 166 166 276 276 276 F F F 22 440-459 168 168 168 280 280 280 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 168 168 168 281 281 281 F F F 25 500-519 169 169 169 282 282 282 F F F 26 520-539 164 164 164 273 273 273 F F F 27 540-559 168 168 168 280 280 280 F F F 28 560-579 168 168 168 280 280 280 F F F 29 580-599 168 168 168 279 279 279 F F F Max gen: 169, Max flush: 282 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.452107309Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.498338417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.499259387Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.499303999Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.499318536Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.499332843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870} {"msg":"Crucible 3d22c6d0-b2d7-480e-83c8-d6091b1f07b4 has session id: aff87fc3-748e-4e13-9b5f-bca657b3c852","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.499972925Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Upstairs opts: Upstairs UUID: 3d22c6d0-b2d7-480e-83c8-d6091b1f07b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500015407Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Crucible stats registered with UUID: 3d22c6d0-b2d7-480e-83c8-d6091b1f07b4","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500040033Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:33.500060949Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500629793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500664287Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870} {"msg":"3d22c6d0-b2d7-480e-83c8-d6091b1f07b4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500686183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500707429Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"0","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500742643Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"0","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500764499Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"1","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500793873Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"1","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500817059Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"2","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500843584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"2","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.50086634Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500888785Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.50092022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"ds_connection connected from Ok(127.0.0.1:58951)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500945135Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"ds_connection connected from Ok(127.0.0.1:60275)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500967231Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"ds_connection connected from Ok(127.0.0.1:53161)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.500989847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.501737608Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"2","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.501779921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"0","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.502317901Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"1","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.502510036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"0","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.5025404Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"2","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.502901233Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"1","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503078191Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503162545Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[0]R flush_numbers[0..12]: [275, 267, 282, 280, 277, 277, 273, 273, 265, 274, 277, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.5034573Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[0]R generation[0..12]: [166, 162, 169, 168, 167, 167, 164, 164, 161, 165, 167, 168]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503482546Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.50351028Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[1]R flush_numbers[0..12]: [275, 267, 282, 280, 277, 277, 273, 273, 265, 274, 277, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503524828Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[1]R generation[0..12]: [166, 162, 169, 168, 167, 167, 164, 164, 161, 165, 167, 168]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503538275Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503551863Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[2]R flush_numbers[0..12]: [275, 267, 282, 280, 277, 277, 273, 273, 265, 274, 277, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.50356628Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[2]R generation[0..12]: [166, 162, 169, 168, 167, 167, 164, 164, 161, 165, 167, 168]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503579898Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503593685Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-07T18:25:33.503606173Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503626809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503643816Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503657633Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"":"downstairs","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503670551Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503682149Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"0","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503694957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"1","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503707584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"client":"2","":"downstairs","session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503720262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.50373219Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"3d22c6d0-b2d7-480e-83c8-d6091b1f07b4 is now active with session: aff87fc3-748e-4e13-9b5f-bca657b3c852","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503744577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503756215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870,"session_id":"aff87fc3-748e-4e13-9b5f-bca657b3c852"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:33.503767953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 353 len 28672 data: 13 13 14 18 19 20 21 02/30 Write block 474 len 12288 data: 15 17 15 03/30 Write block 565 len 16384 data: 17 16 15 14 04/30 Read block 557 len 12288 05/30 Read block 450 len 32768 06/30 Read block 50 len 24576 07/30 Read block 344 len 16384 08/30 Read block 324 len 28672 09/30 Write block 426 len 36864 data: 12 10 9 11 9 10 6 7 9 10/30 Flush 11/30 Write block 117 len 4096 data: 14 12/30 Write block 461 len 20480 data: 9 10 10 12 11 13/30 Write block 160 len 4096 data: 10 14/30 Write block 281 len 32768 data: 12 12 14 14 11 11 10 12 15/30 Read block 20 len 16384 16/30 Read block 189 len 28672 17/30 Write block 249 len 4096 data: 14 18/30 Write block 113 len 4096 data: 16 19/30 Read block 441 len 40960 20/30 Write block 362 len 12288 data: 14 13 13 21/30 Write block 290 len 24576 data: 16 17 12 13 10 12 22/30 Write block 342 len 40960 data: 13 13 19 17 17 17 14 13 13 11 23/30 Write block 309 len 12288 data: 14 17 19 24/30 Write block 507 len 16384 data: 12 10 10 10 25/30 Read block 53 len 40960 26/30 Read block 582 len 36864 27/30 Write block 458 len 40960 data: 12 11 11 10 11 11 13 12 8 9 28/30 Write block 465 len 28672 data: 13 9 10 10 13 10 11 29/30 Read block 187 len 40960 30/30 Write block 553 len 20480 data: 11 10 12 12 13 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 1 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:33.523441844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 275 275 275 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 169 169 169 282 282 282 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 167 167 167 277 277 277 T T T 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 161 161 161 265 265 265 T T T 9 180-199 165 165 165 274 274 274 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 168 168 168 280 280 280 F F F 12 240-259 164 164 164 269 269 269 T T T 13 260-279 169 169 169 282 282 282 F F F 14 280-299 168 168 168 280 280 280 T T T 15 300-319 168 168 168 278 278 278 T T T 16 320-339 164 164 164 269 269 269 F F F 17 340-359 170 170 170 283 283 283 T T T 18 360-379 166 166 166 276 276 276 T T T 19 380-399 168 168 168 278 278 278 F F F 20 400-419 168 168 168 280 280 280 F F F 21 420-439 170 170 170 283 283 283 F F F 22 440-459 168 168 168 280 280 280 T T T 23 460-479 170 170 170 283 283 283 T T T 24 480-499 168 168 168 281 281 281 F F F 25 500-519 169 169 169 282 282 282 T T T 26 520-539 164 164 164 273 273 273 F F F 27 540-559 168 168 168 280 280 280 T T T 28 560-579 170 170 170 283 283 283 F F F 29 580-599 168 168 168 279 279 279 F F F Max gen: 170, Max flush: 283 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 275 275 275 F F F 1 020-039 162 162 162 267 267 267 F F F 2 040-059 169 169 169 282 282 282 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 164 164 164 273 273 273 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 165 165 165 274 274 274 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 168 168 168 280 280 280 F F F 12 240-259 171 171 171 284 284 284 F F F 13 260-279 169 169 169 282 282 282 F F F 14 280-299 171 171 171 284 284 284 F F F 15 300-319 171 171 171 284 284 284 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 171 171 171 284 284 284 F F F 18 360-379 171 171 171 284 284 284 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 168 168 168 280 280 280 F F F 21 420-439 170 170 170 283 283 283 F F F 22 440-459 171 171 171 284 284 284 F F F 23 460-479 171 171 171 284 284 284 F F F 24 480-499 168 168 168 281 281 281 F F F 25 500-519 171 171 171 284 284 284 F F F 26 520-539 164 164 164 273 273 273 F F F 27 540-559 171 171 171 284 284 284 F F F 28 560-579 170 170 170 283 283 283 F F F 29 580-599 168 168 168 279 279 279 F F F Max gen: 171, Max flush: 284 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.813621258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.861673469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.862660276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.862704708Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.862719255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.862733373Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878} {"msg":"Crucible 840b0024-ee9a-4284-a317-139cb54b1d00 has session id: 47204a8c-3599-4054-b819-3d711f444c98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.863438882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Upstairs opts: Upstairs UUID: 840b0024-ee9a-4284-a317-139cb54b1d00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.863490522Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Crucible stats registered with UUID: 840b0024-ee9a-4284-a317-139cb54b1d00","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.863525416Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:47.863548272Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864036592Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864063476Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878} {"msg":"840b0024-ee9a-4284-a317-139cb54b1d00 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864135063Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.86415351Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"0","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864180665Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"0","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864194882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"1","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.86420892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"1","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864221997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"2","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864235915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"2","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864248832Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.86426248Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864276227Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"ds_connection connected from Ok(127.0.0.1:65516)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864442806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"ds_connection connected from Ok(127.0.0.1:56273)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864492007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"ds_connection connected from Ok(127.0.0.1:52535)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.864538119Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.865265974Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"0","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.865302307Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"2","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.865877331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"2","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.866029403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"0","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.866500286Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.866605206Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.866824746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"1","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.867390681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"1","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[0]R flush_numbers[0..12]: [275, 267, 282, 280, 277, 284, 273, 273, 284, 274, 277, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.867978702Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[0]R generation[0..12]: [166, 162, 169, 168, 167, 171, 164, 164, 171, 165, 167, 168]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868017365Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868040251Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[1]R flush_numbers[0..12]: [275, 267, 282, 280, 277, 284, 273, 273, 284, 274, 277, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868058118Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[1]R generation[0..12]: [166, 162, 169, 168, 167, 171, 164, 164, 171, 165, 167, 168]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868080904Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868107968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[2]R flush_numbers[0..12]: [275, 267, 282, 280, 277, 284, 273, 273, 284, 274, 277, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868134603Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[2]R generation[0..12]: [166, 162, 169, 168, 167, 171, 164, 164, 171, 165, 167, 168]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868158839Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868189403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868219848Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868242034Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868264799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868293534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"":"downstairs","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868323739Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868351923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"0","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.86837124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"1","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:25:47.868399625Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"client":"2","":"downstairs","session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868428489Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868453544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"840b0024-ee9a-4284-a317-139cb54b1d00 is now active with session: 47204a8c-3599-4054-b819-3d711f444c98","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868483149Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868524131Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878,"session_id":"47204a8c-3599-4054-b819-3d711f444c98"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:47.868551317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 307 len 24576 data: 12 16 15 18 20 20 02/30 Write block 544 len 16384 data: 17 17 17 15 03/30 Flush 04/30 Write block 408 len 20480 data: 13 13 14 14 15 05/30 Write block 593 len 20480 data: 8 9 9 8 5 06/30 Read block 552 len 4096 07/30 Write block 3 len 28672 data: 11 11 11 13 14 17 22 08/30 Flush 09/30 Write block 187 len 8192 data: 9 7 10/30 Read block 52 len 36864 11/30 Write block 580 len 36864 data: 22 21 20 12 14 11 9 10 10 12/30 Write block 0 len 32768 data: 4 7 11 12 12 12 14 15 13/30 Read block 461 len 32768 14/30 Write block 176 len 4096 data: 17 15/30 Flush 16/30 Write block 163 len 4096 data: 11 17/30 Write block 271 len 36864 data: 13 16 14 11 9 12 14 11 12 18/30 Read block 428 len 4096 19/30 Write block 269 len 40960 data: 11 12 14 17 15 12 10 13 15 12 20/30 Write block 33 len 40960 data: 11 9 7 8 10 10 8 12 9 9 21/30 Write block 223 len 4096 data: 8 22/30 Flush 23/30 Write block 120 len 12288 data: 21 16 12 24/30 Read block 401 len 32768 25/30 Write block 427 len 32768 data: 11 10 12 10 11 7 8 10 26/30 Flush 27/30 Write block 179 len 36864 data: 18 18 17 15 12 12 11 11 10 28/30 Write block 546 len 20480 data: 18 16 16 8 10 29/30 Write block 174 len 16384 data: 16 17 18 17 30/30 Write block 237 len 8192 data: 11 12 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:47.888193733Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 172 172 172 288 288 288 F F F 2 040-059 172 172 172 288 288 288 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 172 172 172 289 289 289 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 172 172 172 288 288 288 T T T 9 180-199 172 172 172 287 287 287 T T T 10 200-219 167 167 167 277 277 277 F F F 11 220-239 172 172 172 288 288 288 T T T 12 240-259 171 171 171 284 284 284 F F F 13 260-279 172 172 172 288 288 288 F F F 14 280-299 171 171 171 284 284 284 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 171 171 171 284 284 284 F F F 18 360-379 171 171 171 284 284 284 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 172 172 172 286 286 286 F F F 21 420-439 172 172 172 289 289 289 F F F 22 440-459 171 171 171 284 284 284 F F F 23 460-479 171 171 171 284 284 284 F F F 24 480-499 168 168 168 281 281 281 F F F 25 500-519 171 171 171 284 284 284 F F F 26 520-539 164 164 164 273 273 273 F F F 27 540-559 172 172 172 285 285 285 T T T 28 560-579 170 170 170 283 283 283 F F F 29 580-599 172 172 172 287 287 287 F F F Max gen: 172, Max flush: 289 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 172 172 172 288 288 288 F F F 2 040-059 172 172 172 288 288 288 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 167 167 167 277 277 277 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 172 172 172 289 289 289 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 173 173 173 290 290 290 F F F 9 180-199 173 173 173 290 290 290 F F F 10 200-219 167 167 167 277 277 277 F F F 11 220-239 173 173 173 290 290 290 F F F 12 240-259 171 171 171 284 284 284 F F F 13 260-279 172 172 172 288 288 288 F F F 14 280-299 171 171 171 284 284 284 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 171 171 171 284 284 284 F F F 18 360-379 171 171 171 284 284 284 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 172 172 172 286 286 286 F F F 21 420-439 172 172 172 289 289 289 F F F 22 440-459 171 171 171 284 284 284 F F F 23 460-479 171 171 171 284 284 284 F F F 24 480-499 168 168 168 281 281 281 F F F 25 500-519 171 171 171 284 284 284 F F F 26 520-539 164 164 164 273 273 273 F F F 27 540-559 173 173 173 290 290 290 F F F 28 560-579 170 170 170 283 283 283 F F F 29 580-599 172 172 172 287 287 287 F F F Max gen: 173, Max flush: 290 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.139493998Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.18483639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.185680084Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.185724286Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.185745802Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.185760539Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886} {"msg":"Crucible 448eb661-5dbe-4348-b861-0120874066e5 has session id: f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.18635466Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Upstairs opts: Upstairs UUID: 448eb661-5dbe-4348-b861-0120874066e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.186386594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Crucible stats registered with UUID: 448eb661-5dbe-4348-b861-0120874066e5","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.186412679Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:52.186425946Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.186930693Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.186963537Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886} {"msg":"448eb661-5dbe-4348-b861-0120874066e5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187022446Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187048021Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"0","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187079706Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"0","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187106321Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187134215Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.18716089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"2","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187197913Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"2","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187222149Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187245905Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187267291Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:55309)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187291486Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:49221)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187316742Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:52.187625114Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:52.187666647Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:25:52.187686703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.187783275Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.188011283Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"0","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.188037208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"2","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.188647845Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"2","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.188760964Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"0","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.189201153Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:25:52.189304234Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.188301589Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:57132)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.188403661Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.189721967Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.190324875Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[0]R flush_numbers[0..12]: [287, 288, 288, 280, 277, 284, 289, 273, 290, 290, 277, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.190938821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[0]R generation[0..12]: [172, 172, 172, 168, 167, 171, 172, 164, 173, 173, 167, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.190964737Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.190983173Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[1]R flush_numbers[0..12]: [287, 288, 288, 280, 277, 284, 289, 273, 290, 290, 277, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.19100007Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[1]R generation[0..12]: [172, 172, 172, 168, 167, 171, 172, 164, 173, 173, 167, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191018387Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191035544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[2]R flush_numbers[0..12]: [287, 288, 288, 280, 277, 284, 289, 273, 290, 290, 277, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.19105264Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"[2]R generation[0..12]: [172, 172, 172, 168, 167, 171, 172, 164, 173, 173, 167, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191069377Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191098942Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191116819Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191133196Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191148963Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.19116433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"":"downstairs","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191180037Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191194284Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"0","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191210611Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"1","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191227218Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"client":"2","":"downstairs","session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191244065Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2025-03-07T18:26:02.191263292Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"448eb661-5dbe-4348-b861-0120874066e5 is now active with session: f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191279178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191293806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886,"session_id":"f4c4401d-0cab-4b3e-8ffb-d6bb72aa6e52"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:02.191308883Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 448 len 40960 data: 8 11 11 9 9 13 12 14 14 15 02/30 Read block 584 len 36864 03/30 Read block 317 len 16384 04/30 Write block 199 len 36864 data: 12 10 12 14 15 15 14 16 16 05/30 Read block 46 len 28672 06/30 Read block 344 len 24576 07/30 Read block 517 len 20480 08/30 Read block 240 len 36864 09/30 Read block 224 len 40960 10/30 Read block 66 len 4096 11/30 Read block 579 len 24576 12/30 Read block 11 len 12288 13/30 Flush 14/30 Write block 521 len 28672 data: 9 11 11 14 13 11 10 15/30 Read block 331 len 24576 16/30 Read block 184 len 20480 17/30 Write block 355 len 20480 data: 15 19 20 21 22 18/30 Read block 140 len 32768 19/30 Read block 361 len 28672 20/30 Write block 285 len 24576 data: 12 12 11 13 15 17 21/30 Read block 357 len 40960 22/30 Write block 88 len 36864 data: 13 12 10 9 10 13 13 12 10 23/30 Write block 419 len 20480 data: 11 11 10 10 13 24/30 Read block 472 len 28672 25/30 Read block 464 len 32768 26/30 Write block 91 len 4096 data: 10 27/30 Write block 53 len 20480 data: 10 10 8 8 8 28/30 Read block 12 len 20480 29/30 Write block 484 len 36864 data: 19 20 21 19 18 13 15 16 17 30/30 Read block 47 len 28672 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:02.215830187Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 172 172 172 288 288 288 F F F 2 040-059 172 172 172 288 288 288 T T T 3 060-079 168 168 168 280 280 280 F F F 4 080-099 167 167 167 277 277 277 T T T 5 100-119 171 171 171 284 284 284 F F F 6 120-139 172 172 172 289 289 289 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 173 173 173 290 290 290 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 174 174 174 291 291 291 F F F 11 220-239 173 173 173 290 290 290 F F F 12 240-259 171 171 171 284 284 284 F F F 13 260-279 172 172 172 288 288 288 F F F 14 280-299 171 171 171 284 284 284 T T T 15 300-319 172 172 172 285 285 285 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 171 171 171 284 284 284 T T T 18 360-379 171 171 171 284 284 284 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 172 172 172 286 286 286 T T T 21 420-439 172 172 172 289 289 289 T T T 22 440-459 174 174 174 291 291 291 F F F 23 460-479 171 171 171 284 284 284 F F F 24 480-499 168 168 168 281 281 281 T T T 25 500-519 171 171 171 284 284 284 F F F 26 520-539 164 164 164 273 273 273 T T T 27 540-559 173 173 173 290 290 290 F F F 28 560-579 170 170 170 283 283 283 F F F 29 580-599 172 172 172 287 287 287 F F F Max gen: 174, Max flush: 291 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 172 172 172 288 288 288 F F F 2 040-059 175 175 175 292 292 292 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 175 175 175 292 292 292 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 172 172 172 289 289 289 F F F 7 140-159 164 164 164 273 273 273 F F F 8 160-179 173 173 173 290 290 290 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 174 174 174 291 291 291 F F F 11 220-239 173 173 173 290 290 290 F F F 12 240-259 171 171 171 284 284 284 F F F 13 260-279 172 172 172 288 288 288 F F F 14 280-299 175 175 175 292 292 292 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 164 164 164 269 269 269 F F F 17 340-359 175 175 175 292 292 292 F F F 18 360-379 171 171 171 284 284 284 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 175 175 175 292 292 292 F F F 21 420-439 175 175 175 292 292 292 F F F 22 440-459 174 174 174 291 291 291 F F F 23 460-479 171 171 171 284 284 284 F F F 24 480-499 175 175 175 292 292 292 F F F 25 500-519 171 171 171 284 284 284 F F F 26 520-539 175 175 175 292 292 292 F F F 27 540-559 173 173 173 290 290 290 F F F 28 560-579 170 170 170 283 283 283 F F F 29 580-599 172 172 172 287 287 287 F F F Max gen: 175, Max flush: 292 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.484794406Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.52975205Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.530651793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.530697715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.530712672Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.530727099Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894} {"msg":"Crucible 29cf8694-3da7-4844-8d53-e4b84ceb7710 has session id: ce3d38fd-871f-4fd9-b7ad-6316102a1eec","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.531326098Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Upstairs opts: Upstairs UUID: 29cf8694-3da7-4844-8d53-e4b84ceb7710, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.531366451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Crucible stats registered with UUID: 29cf8694-3da7-4844-8d53-e4b84ceb7710","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.531389827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:06.531410183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.531934866Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.531969299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894} {"msg":"29cf8694-3da7-4844-8d53-e4b84ceb7710 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532052504Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.53207121Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"0","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532097276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"0","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532111463Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"1","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.5321257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"1","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532138858Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"2","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532152795Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"2","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532165483Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532187169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532213534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"ds_connection connected from Ok(127.0.0.1:64268)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.53229316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"ds_connection connected from Ok(127.0.0.1:64652)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532333772Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"ds_connection connected from Ok(127.0.0.1:62396)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.532361777Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.533029923Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"2","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.533058738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"0","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.53369578Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"0","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.533736133Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"2","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.534396051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.534422815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"1","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.534437953Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.53499075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"1","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[0]R flush_numbers[0..12]: [287, 288, 292, 280, 292, 284, 289, 273, 290, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535522402Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[0]R generation[0..12]: [172, 172, 175, 168, 175, 171, 172, 164, 173, 174, 174, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535548597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535563555Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[1]R flush_numbers[0..12]: [287, 288, 292, 280, 292, 284, 289, 273, 290, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535577022Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[1]R generation[0..12]: [172, 172, 175, 168, 175, 171, 172, 164, 173, 174, 174, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535590199Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535603607Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[2]R flush_numbers[0..12]: [287, 288, 292, 280, 292, 284, 289, 273, 290, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535617105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[2]R generation[0..12]: [172, 172, 175, 168, 175, 171, 172, 164, 173, 174, 174, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535630452Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535644Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535656857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535677793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535690331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535703099Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"":"downstairs","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535715846Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535727474Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"0","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535740462Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"1","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535755579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"client":"2","":"downstairs","session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535768606Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535780544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"29cf8694-3da7-4844-8d53-e4b84ceb7710 is now active with session: ce3d38fd-871f-4fd9-b7ad-6316102a1eec","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535792952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.53580459Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894,"session_id":"ce3d38fd-871f-4fd9-b7ad-6316102a1eec"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:06.535816648Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 270 len 24576 data: 13 15 18 16 13 11 02/30 Read block 355 len 28672 03/30 Write block 131 len 4096 data: 6 04/30 Read block 362 len 28672 05/30 Read block 77 len 32768 06/30 Write block 461 len 4096 data: 11 07/30 Write block 579 len 8192 data: 20 23 08/30 Read block 331 len 12288 09/30 Read block 341 len 20480 10/30 Write block 284 len 32768 data: 15 13 13 12 14 16 18 18 11/30 Read block 518 len 28672 12/30 Read block 199 len 40960 13/30 Write block 34 len 32768 data: 10 8 9 11 11 9 13 10 14/30 Write block 141 len 12288 data: 11 15 9 15/30 Read block 409 len 36864 16/30 Flush 17/30 Write block 239 len 36864 data: 14 11 10 10 12 9 9 13 13 18/30 Read block 500 len 32768 19/30 Read block 32 len 24576 20/30 Flush 21/30 Write block 362 len 36864 data: 15 14 14 12 14 17 16 17 15 22/30 Write block 500 len 36864 data: 16 11 11 13 12 14 12 13 11 23/30 Read block 200 len 32768 24/30 Read block 582 len 12288 25/30 Flush 26/30 Write block 272 len 40960 data: 19 17 14 12 14 16 13 13 14 13 27/30 Write block 160 len 24576 data: 11 11 12 12 12 11 28/30 Write block 162 len 40960 data: 13 13 13 12 11 8 8 10 12 12 29/30 Write block 80 len 20480 data: 15 15 15 13 17 30/30 Write block 327 len 24576 data: 12 9 7 6 8 13 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:06.557168068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 176 176 176 293 293 293 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 175 175 175 292 292 292 T T T 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 176 176 176 293 293 293 F F F 8 160-179 173 173 173 290 290 290 T T T 9 180-199 174 174 174 291 291 291 F F F 10 200-219 174 174 174 291 291 291 F F F 11 220-239 176 176 176 294 294 294 F F F 12 240-259 176 176 176 294 294 294 F F F 13 260-279 176 176 176 293 293 293 T T T 14 280-299 176 176 176 293 293 293 T T T 15 300-319 172 172 172 285 285 285 F F F 16 320-339 164 164 164 269 269 269 T T T 17 340-359 175 175 175 292 292 292 F F F 18 360-379 176 176 176 295 295 295 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 175 175 175 292 292 292 F F F 21 420-439 175 175 175 292 292 292 F F F 22 440-459 174 174 174 291 291 291 F F F 23 460-479 176 176 176 293 293 293 F F F 24 480-499 175 175 175 292 292 292 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 175 175 175 292 292 292 F F F 27 540-559 173 173 173 290 290 290 F F F 28 560-579 176 176 176 293 293 293 F F F 29 580-599 176 176 176 293 293 293 F F F Max gen: 176, Max flush: 295 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 176 176 176 293 293 293 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 177 177 177 296 296 296 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 176 176 176 293 293 293 F F F 8 160-179 177 177 177 296 296 296 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 174 174 174 291 291 291 F F F 11 220-239 176 176 176 294 294 294 F F F 12 240-259 176 176 176 294 294 294 F F F 13 260-279 177 177 177 296 296 296 F F F 14 280-299 177 177 177 296 296 296 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 177 177 177 296 296 296 F F F 17 340-359 175 175 175 292 292 292 F F F 18 360-379 176 176 176 295 295 295 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 175 175 175 292 292 292 F F F 21 420-439 175 175 175 292 292 292 F F F 22 440-459 174 174 174 291 291 291 F F F 23 460-479 176 176 176 293 293 293 F F F 24 480-499 175 175 175 292 292 292 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 175 175 175 292 292 292 F F F 27 540-559 173 173 173 290 290 290 F F F 28 560-579 176 176 176 293 293 293 F F F 29 580-599 176 176 176 293 293 293 F F F Max gen: 177, Max flush: 296 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.815700895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:10.859372257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.859779502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.891267597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.891896831Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.891933194Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.891946731Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.891960969Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902} {"msg":"Crucible 5376bae5-9dc8-4054-9f31-ddc7d71f7a36 has session id: 422170ff-59ed-4d9a-bacb-aff647e0050b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.892491301Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Upstairs opts: Upstairs UUID: 5376bae5-9dc8-4054-9f31-ddc7d71f7a36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.892530043Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Crucible stats registered with UUID: 5376bae5-9dc8-4054-9f31-ddc7d71f7a36","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.89254664Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:10.892559388Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.892976581Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893011125Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902} {"msg":"5376bae5-9dc8-4054-9f31-ddc7d71f7a36 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893076222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893109427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"0","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893138681Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"0","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893153158Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"1","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893167166Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"1","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893180203Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"2","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893203889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"2","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893228105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.89325229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893274866Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43015)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893346223Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58378)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.893383666Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39811)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.89341297Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.894292617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"1","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.894321502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"2","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.89433632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"0","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895026712Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"1","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895066774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"0","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895095919Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"2","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895617763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895731601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[0]R flush_numbers[0..12]: [287, 293, 293, 280, 296, 284, 293, 293, 296, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895809077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[0]R generation[0..12]: [172, 176, 176, 168, 177, 171, 176, 176, 177, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895828683Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895842951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[1]R flush_numbers[0..12]: [287, 293, 293, 280, 296, 284, 293, 293, 296, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895856498Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[1]R generation[0..12]: [172, 176, 176, 168, 177, 171, 176, 176, 177, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895869866Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895883243Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[2]R flush_numbers[0..12]: [287, 293, 293, 280, 296, 284, 293, 293, 296, 291, 291, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895896481Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[2]R generation[0..12]: [172, 176, 176, 168, 177, 171, 176, 176, 177, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895910658Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895924266Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895941363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.89595465Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895974337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.895986834Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"":"downstairs","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896015859Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896028687Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"0","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896041714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"1","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896054432Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"client":"2","":"downstairs","session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896067159Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896079027Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"5376bae5-9dc8-4054-9f31-ddc7d71f7a36 is now active with session: 422170ff-59ed-4d9a-bacb-aff647e0050b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896091315Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.896102993Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902,"session_id":"422170ff-59ed-4d9a-bacb-aff647e0050b"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:10.89611488Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 556 len 28672 data: 13 14 12 14 15 17 15 02/30 Read block 529 len 8192 03/30 Read block 405 len 16384 04/30 Read block 170 len 32768 05/30 Read block 208 len 4096 06/30 Read block 385 len 32768 07/30 Read block 292 len 28672 08/30 Write block 245 len 20480 data: 10 14 14 14 15 09/30 Write block 157 len 16384 data: 17 13 9 12 10/30 Write block 446 len 24576 data: 12 10 9 12 12 10 11/30 Read block 55 len 24576 12/30 Write block 453 len 24576 data: 14 13 15 15 16 13 13/30 Write block 529 len 40960 data: 9 10 11 10 8 6 6 10 15 14 14/30 Write block 567 len 12288 data: 16 15 11 15/30 Read block 381 len 32768 16/30 Read block 579 len 4096 17/30 Read block 124 len 40960 18/30 Read block 280 len 16384 19/30 Write block 541 len 32768 data: 12 16 16 18 18 19 17 17 20/30 Write block 214 len 20480 data: 10 7 10 9 10 21/30 Flush 22/30 Write block 537 len 40960 data: 16 15 14 13 13 17 17 19 19 20 23/30 Read block 146 len 16384 24/30 Flush 25/30 Write block 48 len 12288 data: 11 15 11 26/30 Read block 588 len 40960 27/30 Read block 154 len 20480 28/30 Write block 142 len 40960 data: 16 10 10 13 16 16 15 14 13 14 29/30 Read block 277 len 40960 30/30 Write block 345 len 40960 data: 18 18 18 15 14 14 12 10 14 14 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:10.917780803Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 176 176 176 293 293 293 T T T 3 060-079 168 168 168 280 280 280 F F F 4 080-099 177 177 177 296 296 296 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 178 178 178 297 297 297 T T T 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 178 178 178 297 297 297 F F F 11 220-239 176 176 176 294 294 294 F F F 12 240-259 178 178 178 297 297 297 F F F 13 260-279 177 177 177 296 296 296 F F F 14 280-299 177 177 177 296 296 296 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 177 177 177 296 296 296 F F F 17 340-359 175 175 175 292 292 292 T T T 18 360-379 176 176 176 295 295 295 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 175 175 175 292 292 292 F F F 21 420-439 175 175 175 292 292 292 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 176 176 176 293 293 293 F F F 24 480-499 175 175 175 292 292 292 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 178 178 178 298 298 298 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 178 178 178 297 297 297 F F F 29 580-599 176 176 176 293 293 293 F F F Max gen: 178, Max flush: 298 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 299 299 299 F F F 3 060-079 168 168 168 280 280 280 F F F 4 080-099 177 177 177 296 296 296 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 179 179 179 299 299 299 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 178 178 178 297 297 297 F F F 11 220-239 176 176 176 294 294 294 F F F 12 240-259 178 178 178 297 297 297 F F F 13 260-279 177 177 177 296 296 296 F F F 14 280-299 177 177 177 296 296 296 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 177 177 177 296 296 296 F F F 17 340-359 179 179 179 299 299 299 F F F 18 360-379 176 176 176 295 295 295 F F F 19 380-399 168 168 168 278 278 278 F F F 20 400-419 175 175 175 292 292 292 F F F 21 420-439 175 175 175 292 292 292 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 176 176 176 293 293 293 F F F 24 480-499 175 175 175 292 292 292 F F F 25 500-519 176 176 176 295 295 295 F F F 26 520-539 178 178 178 298 298 298 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 178 178 178 297 297 297 F F F 29 580-599 176 176 176 293 293 293 F F F Max gen: 179, Max flush: 299 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.188422841Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:15.234476772Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.235049876Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.267485006Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.268671236Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.268717948Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.268736535Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.268754581Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910} {"msg":"Crucible be79c363-3fba-4065-bf3e-29d28669a6e9 has session id: 92308994-fed6-4236-abf3-dc082601a3b1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.269497594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Upstairs opts: Upstairs UUID: be79c363-3fba-4065-bf3e-29d28669a6e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.269531228Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Crucible stats registered with UUID: be79c363-3fba-4065-bf3e-29d28669a6e9","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.269546925Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:15.269560002Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270097313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270124268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910} {"msg":"be79c363-3fba-4065-bf3e-29d28669a6e9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270196425Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270216051Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"0","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270242076Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"0","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270255714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"1","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270269671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"1","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270285968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"2","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270312123Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"2","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270336128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270361124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270393908Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53360)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270454147Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57905)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270732185Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40438)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.270763569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.271211727Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"1","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.271343312Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"2","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.271451652Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"0","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.271769933Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"1","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.271902549Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"2","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272190106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"0","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272357175Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272495479Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[0]R flush_numbers[0..12]: [287, 293, 299, 280, 296, 284, 293, 299, 297, 291, 297, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272777957Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[0]R generation[0..12]: [172, 176, 179, 168, 177, 171, 176, 179, 178, 174, 178, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272797403Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.27281646Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[1]R flush_numbers[0..12]: [287, 293, 299, 280, 296, 284, 293, 299, 297, 291, 297, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272830937Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[1]R generation[0..12]: [172, 176, 179, 168, 177, 171, 176, 179, 178, 174, 178, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272844475Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272858112Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[2]R flush_numbers[0..12]: [287, 293, 299, 280, 296, 284, 293, 299, 297, 291, 297, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.27287129Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[2]R generation[0..12]: [172, 176, 179, 168, 177, 171, 176, 179, 178, 174, 178, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272884777Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272898295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272910802Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:26:15.27292351Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272940827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272953484Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"":"downstairs","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272966512Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.27297847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"0","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.272991447Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"1","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.273004265Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"client":"2","":"downstairs","session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.273019952Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.27303216Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"be79c363-3fba-4065-bf3e-29d28669a6e9 is now active with session: 92308994-fed6-4236-abf3-dc082601a3b1","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.273044967Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.273056565Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910,"session_id":"92308994-fed6-4236-abf3-dc082601a3b1"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:15.273068583Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 568 len 8192 data: 16 12 03/30 Read block 234 len 36864 04/30 Read block 97 len 8192 05/30 Flush 06/30 Write block 156 len 8192 data: 15 18 07/30 Flush 08/30 Write block 75 len 24576 data: 16 15 12 12 15 16 09/30 Flush 10/30 Write block 395 len 40960 data: 9 9 9 12 16 18 16 13 13 13 11/30 Read block 166 len 36864 12/30 Write block 67 len 28672 data: 11 13 17 21 22 22 21 13/30 Write block 388 len 12288 data: 13 15 13 14/30 Write block 507 len 16384 data: 14 12 11 11 15/30 Write block 393 len 12288 data: 12 10 10 16/30 Write block 582 len 24576 data: 21 13 15 12 10 11 17/30 Read block 276 len 36864 18/30 Write block 327 len 36864 data: 13 10 8 7 9 14 14 15 14 19/30 Write block 81 len 12288 data: 16 16 14 20/30 Write block 223 len 8192 data: 9 8 21/30 Read block 172 len 36864 22/30 Write block 348 len 32768 data: 16 15 15 13 11 15 15 16 23/30 Write block 245 len 40960 data: 11 15 15 15 16 15 17 17 17 19 24/30 Flush 25/30 Write block 273 len 20480 data: 18 15 13 15 17 26/30 Read block 467 len 20480 27/30 Read block 499 len 24576 28/30 Write block 442 len 16384 data: 14 11 11 7 29/30 Write block 464 len 28672 data: 14 14 10 11 11 14 11 30/30 Write block 469 len 28672 data: 15 12 12 12 12 16 18 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:15.292409755Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 299 299 299 F F F 3 060-079 180 180 180 304 304 304 F F F 4 080-099 180 180 180 304 304 304 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 180 180 180 302 302 302 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 178 178 178 297 297 297 F F F 11 220-239 180 180 180 304 304 304 F F F 12 240-259 180 180 180 304 304 304 F F F 13 260-279 177 177 177 296 296 296 T T T 14 280-299 177 177 177 296 296 296 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 180 180 180 304 304 304 F F F 17 340-359 180 180 180 304 304 304 F F F 18 360-379 176 176 176 295 295 295 F F F 19 380-399 180 180 180 304 304 304 F F F 20 400-419 180 180 180 304 304 304 F F F 21 420-439 175 175 175 292 292 292 F F F 22 440-459 178 178 178 297 297 297 T T T 23 460-479 176 176 176 293 293 293 T T T 24 480-499 175 175 175 292 292 292 F F F 25 500-519 180 180 180 304 304 304 F F F 26 520-539 178 178 178 298 298 298 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 180 180 180 301 301 301 F F F 29 580-599 180 180 180 304 304 304 F F F Max gen: 180, Max flush: 304 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 179 179 179 299 299 299 F F F 3 060-079 180 180 180 304 304 304 F F F 4 080-099 180 180 180 304 304 304 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 180 180 180 302 302 302 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 178 178 178 297 297 297 F F F 11 220-239 180 180 180 304 304 304 F F F 12 240-259 180 180 180 304 304 304 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 177 177 177 296 296 296 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 180 180 180 304 304 304 F F F 17 340-359 180 180 180 304 304 304 F F F 18 360-379 176 176 176 295 295 295 F F F 19 380-399 180 180 180 304 304 304 F F F 20 400-419 180 180 180 304 304 304 F F F 21 420-439 175 175 175 292 292 292 F F F 22 440-459 181 181 181 305 305 305 F F F 23 460-479 181 181 181 305 305 305 F F F 24 480-499 175 175 175 292 292 292 F F F 25 500-519 180 180 180 304 304 304 F F F 26 520-539 178 178 178 298 298 298 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 180 180 180 301 301 301 F F F 29 580-599 180 180 180 304 304 304 F F F Max gen: 181, Max flush: 305 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.573498911Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:19.619100276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.619813554Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.652542669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.653731599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.653773262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.653788169Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.653802046Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918} {"msg":"Crucible 6e157921-af25-4799-b0c1-89eee0d35691 has session id: 34ff8d99-c1cd-437d-8f20-d516cf83d893","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.654500737Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Upstairs opts: Upstairs UUID: 6e157921-af25-4799-b0c1-89eee0d35691, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.654543419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Crucible stats registered with UUID: 6e157921-af25-4799-b0c1-89eee0d35691","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.654566615Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:19.654586381Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.65513298Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655162055Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918} {"msg":"6e157921-af25-4799-b0c1-89eee0d35691 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655226433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655245839Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"0","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655283942Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"0","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655310317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"1","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655336893Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"1","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655363548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"2","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655390013Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"2","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655413938Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655441423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655466809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"ds_connection connected from Ok(127.0.0.1:43139)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655490254Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"ds_connection connected from Ok(127.0.0.1:51707)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.65551367Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"ds_connection connected from Ok(127.0.0.1:57590)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.655536725Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.656350355Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"2","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.656398496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"1","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.65653908Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"0","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.656937836Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"1","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.65697263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"2","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657209286Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"0","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657538835Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657629788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[0]R flush_numbers[0..12]: [287, 293, 299, 304, 304, 284, 293, 302, 297, 291, 297, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657803686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[0]R generation[0..12]: [172, 176, 179, 180, 180, 171, 176, 180, 178, 174, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657822743Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.65783704Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[1]R flush_numbers[0..12]: [287, 293, 299, 304, 304, 284, 293, 302, 297, 291, 297, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657850697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[1]R generation[0..12]: [172, 176, 179, 180, 180, 171, 176, 180, 178, 174, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657864255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657877872Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[2]R flush_numbers[0..12]: [287, 293, 299, 304, 304, 284, 293, 302, 297, 291, 297, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.65789286Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[2]R generation[0..12]: [172, 176, 179, 180, 180, 171, 176, 180, 178, 174, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657921064Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657935282Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657952419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657965806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.657978494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"All extents match"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.65799954Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"":"downstairs","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658012438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658024145Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"0","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658037283Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"1","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658050331Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"client":"2","":"downstairs","session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658063518Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658075586Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"6e157921-af25-4799-b0c1-89eee0d35691 is now active with session: 34ff8d99-c1cd-437d-8f20-d516cf83d893","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658088053Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658099671Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918,"session_id":"34ff8d99-c1cd-437d-8f20-d516cf83d893"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:19.658111629Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 51 len 8192 data: 11 11 02/30 Read block 97 len 36864 03/30 Flush 04/30 Write block 49 len 36864 data: 16 12 12 12 11 11 9 9 9 05/30 Read block 303 len 12288 06/30 Write block 387 len 24576 data: 13 14 16 14 14 14 07/30 Read block 482 len 12288 08/30 Write block 532 len 20480 data: 11 9 7 7 11 09/30 Read block 586 len 40960 10/30 Read block 333 len 24576 11/30 Write block 213 len 20480 data: 11 11 8 11 10 12/30 Read block 400 len 8192 13/30 Read block 364 len 28672 14/30 Write block 82 len 8192 data: 17 15 15/30 Read block 562 len 24576 16/30 Read block 502 len 8192 17/30 Read block 510 len 40960 18/30 Write block 98 len 8192 data: 10 11 19/30 Read block 236 len 24576 20/30 Read block 259 len 36864 21/30 Write block 247 len 32768 data: 16 16 17 16 18 18 18 20 22/30 Flush 23/30 Write block 487 len 40960 data: 20 19 14 16 17 18 16 16 16 17 24/30 Read block 214 len 24576 25/30 Write block 437 len 8192 data: 13 13 26/30 Flush 27/30 Write block 578 len 36864 data: 18 21 24 22 22 14 16 13 11 28/30 Read block 160 len 20480 29/30 Read block 156 len 12288 30/30 Read block 18 len 24576 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:19.680285638Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 182 182 182 307 307 307 F F F 3 060-079 180 180 180 304 304 304 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 180 180 180 302 302 302 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 180 180 180 304 304 304 F F F 12 240-259 182 182 182 307 307 307 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 177 177 177 296 296 296 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 180 180 180 304 304 304 F F F 17 340-359 180 180 180 304 304 304 F F F 18 360-379 176 176 176 295 295 295 F F F 19 380-399 182 182 182 307 307 307 F F F 20 400-419 180 180 180 304 304 304 F F F 21 420-439 182 182 182 308 308 308 F F F 22 440-459 181 181 181 305 305 305 F F F 23 460-479 181 181 181 305 305 305 F F F 24 480-499 182 182 182 308 308 308 F F F 25 500-519 180 180 180 304 304 304 F F F 26 520-539 182 182 182 307 307 307 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 180 180 180 301 301 301 T T T 29 580-599 180 180 180 304 304 304 T T T Max gen: 182, Max flush: 308 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 182 182 182 307 307 307 F F F 3 060-079 180 180 180 304 304 304 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 180 180 180 302 302 302 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 180 180 180 304 304 304 F F F 12 240-259 182 182 182 307 307 307 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 177 177 177 296 296 296 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 180 180 180 304 304 304 F F F 17 340-359 180 180 180 304 304 304 F F F 18 360-379 176 176 176 295 295 295 F F F 19 380-399 182 182 182 307 307 307 F F F 20 400-419 180 180 180 304 304 304 F F F 21 420-439 182 182 182 308 308 308 F F F 22 440-459 181 181 181 305 305 305 F F F 23 460-479 181 181 181 305 305 305 F F F 24 480-499 182 182 182 308 308 308 F F F 25 500-519 180 180 180 304 304 304 F F F 26 520-539 182 182 182 307 307 307 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 183 183 183 309 309 309 F F F Max gen: 183, Max flush: 309 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:23.958175745Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.005204075Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.006553956Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.006601857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.006617814Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.006632071Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926} {"msg":"Crucible a53482fe-39ed-4257-874b-5499f068af57 has session id: 63b8813b-0c86-4dfc-9cec-99700fafa27c","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007242898Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Upstairs opts: Upstairs UUID: a53482fe-39ed-4257-874b-5499f068af57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007274102Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Crucible stats registered with UUID: a53482fe-39ed-4257-874b-5499f068af57","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007289089Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:24.007317714Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007788697Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007821931Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926} {"msg":"a53482fe-39ed-4257-874b-5499f068af57 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.00788222Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007909105Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"0","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.00793715Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"0","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007951847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007972223Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.007997409Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"2","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.008025893Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"2","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.00804483Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.008070505Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.00809741Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65305)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.008124525Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60468)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.00814947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:24.008178505Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:24.008214338Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:24.008491577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.008642439Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.00906873Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"2","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.009108913Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"0","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.009790317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"0","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.00982843Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"2","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.01042196Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:24.010490077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.009892518Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60787)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.010047699Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.011197826Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.011872262Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[0]R flush_numbers[0..12]: [287, 293, 307, 304, 307, 284, 293, 302, 297, 291, 307, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012528351Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[0]R generation[0..12]: [172, 176, 182, 180, 182, 171, 176, 180, 178, 174, 182, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012564844Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.01258209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[1]R flush_numbers[0..12]: [287, 293, 307, 304, 307, 284, 293, 302, 297, 291, 307, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012596768Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[1]R generation[0..12]: [172, 176, 182, 180, 182, 171, 176, 180, 178, 174, 182, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012610675Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012624763Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[2]R flush_numbers[0..12]: [287, 293, 307, 304, 307, 284, 293, 302, 297, 291, 307, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.01263848Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"[2]R generation[0..12]: [172, 176, 182, 180, 182, 171, 176, 180, 178, 174, 182, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012652088Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012672434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012685302Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012698229Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012710997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012723544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"":"downstairs","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012736232Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.01274794Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"0","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012764257Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"1","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012777554Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"client":"2","":"downstairs","session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012790732Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.01280268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"a53482fe-39ed-4257-874b-5499f068af57 is now active with session: 63b8813b-0c86-4dfc-9cec-99700fafa27c","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-07T18:26:34.012815177Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012828445Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926,"session_id":"63b8813b-0c86-4dfc-9cec-99700fafa27c"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:34.012840693Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 16384 data: 16 17 14 12 02/30 Read block 128 len 8192 03/30 Flush 04/30 Write block 283 len 16384 data: 15 16 14 14 05/30 Write block 532 len 12288 data: 12 10 8 06/30 Write block 462 len 28672 data: 12 12 15 15 11 12 12 07/30 Write block 146 len 20480 data: 17 17 16 15 14 08/30 Read block 44 len 20480 09/30 Write block 377 len 36864 data: 15 15 13 14 12 13 14 14 12 10/30 Read block 132 len 24576 11/30 Read block 29 len 4096 12/30 Read block 321 len 24576 13/30 Flush 14/30 Write block 349 len 8192 data: 16 16 15/30 Read block 467 len 24576 16/30 Write block 404 len 4096 data: 14 17/30 Write block 494 len 20480 data: 17 17 18 15 14 18/30 Read block 251 len 36864 19/30 Read block 524 len 28672 20/30 Write block 245 len 28672 data: 12 16 17 17 18 17 19 21/30 Read block 350 len 8192 22/30 Read block 167 len 20480 23/30 Read block 540 len 40960 24/30 Write block 520 len 36864 data: 11 10 12 12 15 14 12 11 10 25/30 Write block 352 len 24576 data: 12 16 16 17 20 21 26/30 Flush 27/30 Write block 254 len 24576 data: 21 19 15 15 16 15 28/30 Write block 510 len 40960 data: 12 11 10 11 12 12 13 12 11 11 29/30 Write block 73 len 28672 data: 22 21 17 16 13 13 16 30/30 Read block 406 len 8192 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 3 3 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:34.03323395Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 182 182 182 307 307 307 F F F 3 060-079 180 180 180 304 304 304 T T T 4 080-099 182 182 182 307 307 307 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 184 184 184 311 311 311 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 180 180 180 304 304 304 F F F 12 240-259 184 184 184 312 312 312 T T T 13 260-279 181 181 181 305 305 305 F F F 14 280-299 184 184 184 311 311 311 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 180 180 180 304 304 304 F F F 17 340-359 184 184 184 312 312 312 F F F 18 360-379 184 184 184 311 311 311 F F F 19 380-399 184 184 184 311 311 311 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 182 182 182 308 308 308 F F F 22 440-459 184 184 184 310 310 310 F F F 23 460-479 184 184 184 311 311 311 F F F 24 480-499 184 184 184 312 312 312 F F F 25 500-519 180 180 180 304 304 304 T T T 26 520-539 184 184 184 312 312 312 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 183 183 183 309 309 309 F F F Max gen: 184, Max flush: 312 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 182 182 182 307 307 307 F F F 3 060-079 185 185 185 313 313 313 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 171 171 171 284 284 284 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 184 184 184 311 311 311 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 180 180 180 304 304 304 F F F 12 240-259 185 185 185 313 313 313 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 184 184 184 311 311 311 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 180 180 180 304 304 304 F F F 17 340-359 184 184 184 312 312 312 F F F 18 360-379 184 184 184 311 311 311 F F F 19 380-399 184 184 184 311 311 311 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 182 182 182 308 308 308 F F F 22 440-459 184 184 184 310 310 310 F F F 23 460-479 184 184 184 311 311 311 F F F 24 480-499 184 184 184 312 312 312 F F F 25 500-519 185 185 185 313 313 313 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 183 183 183 309 309 309 F F F Max gen: 185, Max flush: 313 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.283604968Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.330471809Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.331507587Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.331560527Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.331578464Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.331596241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934} {"msg":"Crucible a15a63d9-be34-42d7-900b-08c61bcfbfd7 has session id: 55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.332212886Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Upstairs opts: Upstairs UUID: a15a63d9-be34-42d7-900b-08c61bcfbfd7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.332259528Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Crucible stats registered with UUID: a15a63d9-be34-42d7-900b-08c61bcfbfd7","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.332284573Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:38.332304749Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.33289595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.332925205Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934} {"msg":"a15a63d9-be34-42d7-900b-08c61bcfbfd7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333011639Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333037894Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"0","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333067128Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"0","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333087765Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"1","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333103572Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"1","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333117829Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"2","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333132346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"2","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333145694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333159411Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333279299Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53611)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333306134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39187)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.3333301Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58360)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.333356795Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.334156167Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"2","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.334187121Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"0","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.334887062Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"0","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.334931813Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"2","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.335420523Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"1","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.33559914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.335629375Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.335958914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"1","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[0]R flush_numbers[0..12]: [287, 293, 307, 313, 307, 284, 293, 311, 297, 291, 307, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.33651862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[0]R generation[0..12]: [172, 176, 182, 185, 182, 171, 176, 184, 178, 174, 182, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336539116Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336553433Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[1]R flush_numbers[0..12]: [287, 293, 307, 313, 307, 284, 293, 311, 297, 291, 307, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336567241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[1]R generation[0..12]: [172, 176, 182, 185, 182, 171, 176, 184, 178, 174, 182, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336580508Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336594156Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[2]R flush_numbers[0..12]: [287, 293, 307, 313, 307, 284, 293, 311, 297, 291, 307, 304]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336607574Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[2]R generation[0..12]: [172, 176, 182, 185, 182, 171, 176, 184, 178, 174, 182, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336620961Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336634569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336647246Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336664363Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.33667737Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336690068Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336712024Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336723822Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"0","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336736939Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"1","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336749787Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"client":"2","":"downstairs","session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336762745Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336774623Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"a15a63d9-be34-42d7-900b-08c61bcfbfd7 is now active with session: 55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.33678716Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336798718Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934,"session_id":"55bce954-6e6a-4e0c-9f2f-e6ff8fa17c0b"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:38.336810686Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 488 len 24576 data: 20 15 17 18 19 17 02/30 Write block 156 len 16384 data: 16 19 14 10 03/30 Read block 19 len 20480 04/30 Read block 307 len 28672 05/30 Write block 236 len 16384 data: 13 12 13 15 06/30 Read block 27 len 32768 07/30 Read block 440 len 8192 08/30 Read block 218 len 20480 09/30 Read block 550 len 32768 10/30 Write block 168 len 8192 data: 9 11 11/30 Flush 12/30 Flush 13/30 Write block 106 len 12288 data: 13 15 19 14/30 Write block 431 len 12288 data: 12 8 9 15/30 Write block 160 len 8192 data: 13 12 16/30 Read block 494 len 20480 17/30 Write block 353 len 36864 data: 17 17 18 21 22 22 23 19 18 18/30 Read block 312 len 28672 19/30 Read block 18 len 16384 20/30 Flush 21/30 Write block 66 len 20480 data: 13 12 14 18 22 22/30 Read block 184 len 32768 23/30 Read block 333 len 32768 24/30 Read block 81 len 8192 25/30 Read block 496 len 36864 26/30 Read block 121 len 40960 27/30 Write block 321 len 28672 data: 16 14 14 12 12 13 14 28/30 Write block 456 len 16384 data: 17 18 15 13 29/30 Read block 360 len 32768 30/30 Write block 16 len 12288 data: 14 15 16 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:38.357813241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 287 287 287 T T T 1 020-039 176 176 176 293 293 293 F F F 2 040-059 182 182 182 307 307 307 F F F 3 060-079 185 185 185 313 313 313 T T T 4 080-099 182 182 182 307 307 307 F F F 5 100-119 186 186 186 316 316 316 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 186 186 186 316 316 316 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 186 186 186 314 314 314 F F F 12 240-259 185 185 185 313 313 313 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 184 184 184 311 311 311 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 180 180 180 304 304 304 T T T 17 340-359 186 186 186 316 316 316 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 184 184 184 311 311 311 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 186 186 186 316 316 316 F F F 22 440-459 184 184 184 310 310 310 T T T 23 460-479 184 184 184 311 311 311 F F F 24 480-499 186 186 186 314 314 314 F F F 25 500-519 185 185 185 313 313 313 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 183 183 183 309 309 309 F F F Max gen: 186, Max flush: 316 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 317 317 317 F F F 1 020-039 176 176 176 293 293 293 F F F 2 040-059 182 182 182 307 307 307 F F F 3 060-079 187 187 187 317 317 317 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 186 186 186 316 316 316 F F F 6 120-139 176 176 176 293 293 293 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 186 186 186 316 316 316 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 186 186 186 314 314 314 F F F 12 240-259 185 185 185 313 313 313 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 184 184 184 311 311 311 F F F 15 300-319 172 172 172 285 285 285 F F F 16 320-339 187 187 187 317 317 317 F F F 17 340-359 186 186 186 316 316 316 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 184 184 184 311 311 311 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 186 186 186 316 316 316 F F F 22 440-459 187 187 187 317 317 317 F F F 23 460-479 184 184 184 311 311 311 F F F 24 480-499 186 186 186 314 314 314 F F F 25 500-519 185 185 185 313 313 313 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 178 178 178 298 298 298 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 183 183 183 309 309 309 F F F Max gen: 187, Max flush: 317 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.610585345Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.656043476Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.656953657Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.657024344Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.65704564Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.657064087Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942} {"msg":"Crucible fee8fd7d-ccf3-4332-bcf2-6f8335ae905a has session id: 9829b6a7-cb4c-431c-abf3-72755bcc993b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.657645Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Upstairs opts: Upstairs UUID: fee8fd7d-ccf3-4332-bcf2-6f8335ae905a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.657680603Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Crucible stats registered with UUID: fee8fd7d-ccf3-4332-bcf2-6f8335ae905a","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.657701759Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:42.657786134Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658280002Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658313066Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942} {"msg":"fee8fd7d-ccf3-4332-bcf2-6f8335ae905a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658388832Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658406669Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"0","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658433264Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"0","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658447091Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"1","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658461958Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"1","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658475426Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658489183Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658501921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658515288Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658528346Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48804)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658695625Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:42.658739427Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57352)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.658857905Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:42.658892359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:42.658918774Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.65893889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.65937437Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"0","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.659402424Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"1","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.660031038Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"1","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.660060333Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"0","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.660632187Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:42.660662641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.659597438Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59892)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.659707208Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.660902417Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.661476041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[0]R flush_numbers[0..12]: [317, 293, 307, 317, 307, 316, 293, 314, 316, 291, 307, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662093767Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[0]R generation[0..12]: [187, 176, 182, 187, 182, 186, 176, 186, 186, 174, 182, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662118582Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662133829Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[1]R flush_numbers[0..12]: [317, 293, 307, 317, 307, 316, 293, 314, 316, 291, 307, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662147956Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[1]R generation[0..12]: [187, 176, 182, 187, 182, 186, 176, 186, 186, 174, 182, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662163014Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662176871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[2]R flush_numbers[0..12]: [317, 293, 307, 317, 307, 316, 293, 314, 316, 291, 307, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662190579Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[2]R generation[0..12]: [187, 176, 182, 187, 182, 186, 176, 186, 186, 174, 182, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662204016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662217764Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662230451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662243209Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662255747Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662268274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662295599Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662324174Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"0","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662350889Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"1","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662376384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"client":"2","":"downstairs","session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662402769Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662427195Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"fee8fd7d-ccf3-4332-bcf2-6f8335ae905a is now active with session: 9829b6a7-cb4c-431c-abf3-72755bcc993b","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.66245276Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662475626Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942,"session_id":"9829b6a7-cb4c-431c-abf3-72755bcc993b"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:52.662498692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 246 len 32768 data: 17 18 18 19 18 20 19 19 02/30 Read block 394 len 24576 03/30 Read block 343 len 32768 04/30 Read block 437 len 36864 05/30 Flush 06/30 Write block 15 len 24576 data: 17 15 16 17 15 16 07/30 Write block 548 len 16384 data: 18 9 11 9 08/30 Read block 440 len 4096 09/30 Flush 10/30 Write block 322 len 8192 data: 15 15 11/30 Read block 115 len 32768 12/30 Read block 461 len 8192 13/30 Write block 132 len 28672 data: 7 9 6 10 11 14 14 14/30 Read block 62 len 12288 15/30 Flush 16/30 Flush 17/30 Write block 551 len 20480 data: 10 10 12 11 13 18/30 Read block 479 len 40960 19/30 Write block 308 len 8192 data: 17 16 20/30 Read block 432 len 36864 21/30 Write block 167 len 40960 data: 9 10 12 13 13 14 14 17 18 19 22/30 Read block 219 len 32768 23/30 Read block 345 len 36864 24/30 Write block 297 len 16384 data: 12 8 8 9 25/30 Write block 53 len 24576 data: 12 12 10 10 10 7 26/30 Write block 107 len 40960 data: 16 20 19 19 16 14 17 16 17 17 27/30 Read block 260 len 32768 28/30 Read block 357 len 32768 29/30 Read block 473 len 20480 30/30 Read block 342 len 20480 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:52.684173742Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 188 188 188 319 319 319 F F F 2 040-059 182 182 182 307 307 307 T T T 3 060-079 187 187 187 317 317 317 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 186 186 186 316 316 316 T T T 6 120-139 188 188 188 320 320 320 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 186 186 186 316 316 316 T T T 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 186 186 186 314 314 314 F F F 12 240-259 188 188 188 318 318 318 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 184 184 184 311 311 311 T T T 15 300-319 172 172 172 285 285 285 T T T 16 320-339 188 188 188 320 320 320 F F F 17 340-359 186 186 186 316 316 316 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 184 184 184 311 311 311 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 186 186 186 316 316 316 F F F 22 440-459 187 187 187 317 317 317 F F F 23 460-479 184 184 184 311 311 311 F F F 24 480-499 186 186 186 314 314 314 F F F 25 500-519 185 185 185 313 313 313 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 188 188 188 319 319 319 T T T 28 560-579 183 183 183 309 309 309 F F F 29 580-599 183 183 183 309 309 309 F F F Max gen: 188, Max flush: 320 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 188 188 188 319 319 319 F F F 2 040-059 189 189 189 321 321 321 F F F 3 060-079 187 187 187 317 317 317 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 189 189 189 321 321 321 F F F 6 120-139 188 188 188 320 320 320 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 189 189 189 321 321 321 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 186 186 186 314 314 314 F F F 12 240-259 188 188 188 318 318 318 F F F 13 260-279 181 181 181 305 305 305 F F F 14 280-299 189 189 189 321 321 321 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 188 188 188 320 320 320 F F F 17 340-359 186 186 186 316 316 316 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 184 184 184 311 311 311 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 186 186 186 316 316 316 F F F 22 440-459 187 187 187 317 317 317 F F F 23 460-479 184 184 184 311 311 311 F F F 24 480-499 186 186 186 314 314 314 F F F 25 500-519 185 185 185 313 313 313 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 189 189 189 321 321 321 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 183 183 183 309 309 309 F F F Max gen: 189, Max flush: 321 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.940340947Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.985447474Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.986321892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.986367293Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.986381951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.986396538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950} {"msg":"Crucible fd7e3fb3-8041-4585-8b3a-784af131b2c6 has session id: 439ae7fc-4670-45b6-9a3d-a3c1182dfb85","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.9869821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Upstairs opts: Upstairs UUID: fd7e3fb3-8041-4585-8b3a-784af131b2c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987021832Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Crucible stats registered with UUID: fd7e3fb3-8041-4585-8b3a-784af131b2c6","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987045028Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:56.987067494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987554044Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987592717Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950} {"msg":"fd7e3fb3-8041-4585-8b3a-784af131b2c6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987663364Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987690469Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"0","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987723353Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"0","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987750078Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987777533Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987805178Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"2","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987834592Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"2","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987862077Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987887332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.987913168Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"ds_connection connected from Ok(127.0.0.1:52462)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.988046663Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"ds_connection connected from Ok(127.0.0.1:56362)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.988083126Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:56.988110081Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:56.988137496Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:26:56.988264742Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.988348577Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.988824019Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"2","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.988863862Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"0","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.989505423Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"0","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.989546485Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"2","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.990181098Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:26:56.990294337Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.989760996Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"ds_connection connected from Ok(127.0.0.1:43668)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.989874845Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.990993558Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.991594357Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[0]R flush_numbers[0..12]: [319, 319, 321, 317, 307, 321, 320, 314, 321, 291, 307, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992273111Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[0]R generation[0..12]: [188, 188, 189, 187, 182, 189, 188, 186, 189, 174, 182, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992296857Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992311964Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[1]R flush_numbers[0..12]: [319, 319, 321, 317, 307, 321, 320, 314, 321, 291, 307, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.99233253Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[1]R generation[0..12]: [188, 188, 189, 187, 182, 189, 188, 186, 189, 174, 182, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992346808Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992360805Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[2]R flush_numbers[0..12]: [319, 319, 321, 317, 307, 321, 320, 314, 321, 291, 307, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992374263Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[2]R generation[0..12]: [188, 188, 189, 187, 182, 189, 188, 186, 189, 174, 182, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.9923877Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992401617Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992415155Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992428492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-07T18:27:06.992450359Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992463446Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"":"downstairs","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992476454Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992488241Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"0","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992501199Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"1","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992513967Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"client":"2","":"downstairs","session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992530474Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992544071Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"fd7e3fb3-8041-4585-8b3a-784af131b2c6 is now active with session: 439ae7fc-4670-45b6-9a3d-a3c1182dfb85","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992556819Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992568437Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950,"session_id":"439ae7fc-4670-45b6-9a3d-a3c1182dfb85"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:06.992580415Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 256 len 40960 data: 16 16 17 16 13 15 14 11 9 11 03/30 Read block 83 len 32768 04/30 Read block 345 len 36864 05/30 Read block 77 len 16384 06/30 Write block 457 len 12288 data: 19 16 14 07/30 Read block 488 len 8192 08/30 Flush 09/30 Write block 506 len 24576 data: 13 15 13 12 13 12 10/30 Read block 582 len 40960 11/30 Write block 595 len 4096 data: 10 12/30 Flush 13/30 Write block 393 len 16384 data: 13 11 11 10 14/30 Write block 24 len 20480 data: 15 15 11 10 8 15/30 Read block 196 len 28672 16/30 Read block 178 len 12288 17/30 Write block 357 len 12288 data: 23 23 24 18/30 Write block 233 len 36864 data: 16 11 12 14 13 14 16 12 11 19/30 Read block 142 len 40960 20/30 Write block 271 len 28672 data: 16 20 19 16 14 16 18 21/30 Write block 492 len 36864 data: 20 18 18 18 19 16 15 16 17 22/30 Read block 549 len 28672 23/30 Read block 125 len 16384 24/30 Flush 25/30 Write block 113 len 16384 data: 18 17 18 18 26/30 Write block 448 len 20480 data: 10 13 13 11 10 27/30 Read block 551 len 4096 28/30 Read block 268 len 32768 29/30 Write block 421 len 20480 data: 11 11 14 16 11 30/30 Read block 11 len 20480 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 4 6 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:07.014171601Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 189 189 189 321 321 321 F F F 3 060-079 187 187 187 317 317 317 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 189 189 189 321 321 321 T T T 6 120-139 188 188 188 320 320 320 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 189 189 189 321 321 321 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 190 190 190 325 325 325 F F F 12 240-259 190 190 190 325 325 325 F F F 13 260-279 190 190 190 325 325 325 F F F 14 280-299 189 189 189 321 321 321 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 188 188 188 320 320 320 F F F 17 340-359 190 190 190 325 325 325 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 190 190 190 325 325 325 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 186 186 186 316 316 316 T T T 22 440-459 190 190 190 323 323 323 T T T 23 460-479 184 184 184 311 311 311 F F F 24 480-499 190 190 190 325 325 325 F F F 25 500-519 190 190 190 325 325 325 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 189 189 189 321 321 321 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 190 190 190 324 324 324 F F F Max gen: 190, Max flush: 325 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 189 189 189 321 321 321 F F F 3 060-079 187 187 187 317 317 317 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 191 191 191 326 326 326 F F F 6 120-139 188 188 188 320 320 320 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 189 189 189 321 321 321 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 190 190 190 325 325 325 F F F 12 240-259 190 190 190 325 325 325 F F F 13 260-279 190 190 190 325 325 325 F F F 14 280-299 189 189 189 321 321 321 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 188 188 188 320 320 320 F F F 17 340-359 190 190 190 325 325 325 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 190 190 190 325 325 325 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 191 191 191 326 326 326 F F F 23 460-479 184 184 184 311 311 311 F F F 24 480-499 190 190 190 325 325 325 F F F 25 500-519 190 190 190 325 325 325 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 189 189 189 321 321 321 F F F 28 560-579 183 183 183 309 309 309 F F F 29 580-599 190 190 190 324 324 324 F F F Max gen: 191, Max flush: 326 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.260076085Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:11.303817233Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.304391277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.336413544Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.337323695Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.337362298Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.337376295Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.337390683Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958} {"msg":"Crucible 7601bedf-ebfd-4bff-a0a9-8c7e5bbeb095 has session id: 703ca8c1-66bc-4df6-9448-9fd839a6ec64","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.337978594Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Upstairs opts: Upstairs UUID: 7601bedf-ebfd-4bff-a0a9-8c7e5bbeb095, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338008179Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Crucible stats registered with UUID: 7601bedf-ebfd-4bff-a0a9-8c7e5bbeb095","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338030374Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:11.338043452Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338475992Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338500738Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958} {"msg":"7601bedf-ebfd-4bff-a0a9-8c7e5bbeb095 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338590921Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338618186Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"0","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.3386467Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"0","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338667227Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"1","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338682434Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"1","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338695622Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"2","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338709569Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"2","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338724106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338747202Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338773277Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"ds_connection connected from Ok(127.0.0.1:33211)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338798172Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"ds_connection connected from Ok(127.0.0.1:46230)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338835106Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"ds_connection connected from Ok(127.0.0.1:45237)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.338862871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.33951451Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"1","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.339548124Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"2","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.339809585Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"0","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.340308343Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"1","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.340355455Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"2","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.340383289Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"0","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.340940316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.340965122Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[0]R flush_numbers[0..12]: [319, 325, 321, 317, 307, 326, 320, 314, 321, 291, 307, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.340982788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[0]R generation[0..12]: [188, 190, 189, 187, 182, 191, 188, 186, 189, 174, 182, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.340998016Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341012093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[1]R flush_numbers[0..12]: [319, 325, 321, 317, 307, 326, 320, 314, 321, 291, 307, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.34102584Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[1]R generation[0..12]: [188, 190, 189, 187, 182, 191, 188, 186, 189, 174, 182, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341039468Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341053045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[2]R flush_numbers[0..12]: [319, 325, 321, 317, 307, 326, 320, 314, 321, 291, 307, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341066553Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[2]R generation[0..12]: [188, 190, 189, 187, 182, 191, 188, 186, 189, 174, 182, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341079801Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341093378Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341105746Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341122373Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2025-03-07T18:27:11.34113824Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341165175Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"":"downstairs","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341179932Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.34119165Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"0","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341204527Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"1","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341217185Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"client":"2","":"downstairs","session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341229973Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341241871Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"7601bedf-ebfd-4bff-a0a9-8c7e5bbeb095 is now active with session: 703ca8c1-66bc-4df6-9448-9fd839a6ec64","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341254348Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341265836Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958,"session_id":"703ca8c1-66bc-4df6-9448-9fd839a6ec64"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:11.341277724Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 458 len 32768 data: 17 15 12 12 13 13 16 16 02/30 Read block 37 len 32768 03/30 Read block 49 len 20480 04/30 Read block 120 len 24576 05/30 Read block 564 len 40960 06/30 Write block 549 len 40960 data: 10 12 11 11 13 12 14 14 15 13 07/30 Read block 12 len 8192 08/30 Write block 547 len 20480 data: 18 19 11 13 12 09/30 Write block 489 len 4096 data: 16 10/30 Write block 354 len 12288 data: 18 19 22 11/30 Read block 231 len 20480 12/30 Flush 13/30 Flush 14/30 Write block 173 len 20480 data: 15 18 19 20 18 15/30 Read block 302 len 4096 16/30 Read block 553 len 20480 17/30 Read block 1 len 8192 18/30 Read block 103 len 36864 19/30 Read block 236 len 32768 20/30 Write block 572 len 32768 data: 11 14 16 16 14 16 19 22 21/30 Read block 402 len 20480 22/30 Read block 50 len 40960 23/30 Write block 464 len 12288 data: 17 17 12 24/30 Write block 235 len 36864 data: 13 15 14 15 17 13 12 11 13 25/30 Write block 128 len 40960 data: 11 11 9 7 8 10 7 11 12 15 26/30 Read block 450 len 40960 27/30 Write block 54 len 8192 data: 13 11 28/30 Read block 569 len 36864 29/30 Read block 326 len 12288 30/30 Write block 263 len 20480 data: 12 10 12 13 13 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 5 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:11.362852163Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 189 189 189 321 321 321 T T T 3 060-079 187 187 187 317 317 317 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 191 191 191 326 326 326 F F F 6 120-139 188 188 188 320 320 320 T T T 7 140-159 186 186 186 314 314 314 F F F 8 160-179 189 189 189 321 321 321 T T T 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 190 190 190 325 325 325 T T T 12 240-259 190 190 190 325 325 325 T T T 13 260-279 190 190 190 325 325 325 T T T 14 280-299 189 189 189 321 321 321 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 188 188 188 320 320 320 F F F 17 340-359 192 192 192 327 327 327 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 190 190 190 325 325 325 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 192 192 192 327 327 327 T T T 24 480-499 192 192 192 327 327 327 F F F 25 500-519 190 190 190 325 325 325 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 192 192 192 327 327 327 F F F 28 560-579 183 183 183 309 309 309 T T T 29 580-599 190 190 190 324 324 324 F F F Max gen: 192, Max flush: 327 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 193 193 193 328 328 328 F F F 3 060-079 187 187 187 317 317 317 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 191 191 191 326 326 326 F F F 6 120-139 193 193 193 328 328 328 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 193 193 193 328 328 328 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 193 193 193 328 328 328 F F F 12 240-259 193 193 193 328 328 328 F F F 13 260-279 193 193 193 328 328 328 F F F 14 280-299 189 189 189 321 321 321 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 188 188 188 320 320 320 F F F 17 340-359 192 192 192 327 327 327 F F F 18 360-379 186 186 186 316 316 316 F F F 19 380-399 190 190 190 325 325 325 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 193 193 193 328 328 328 F F F 24 480-499 192 192 192 327 327 327 F F F 25 500-519 190 190 190 325 325 325 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 192 192 192 327 327 327 F F F 28 560-579 193 193 193 328 328 328 F F F 29 580-599 190 190 190 324 324 324 F F F Max gen: 193, Max flush: 328 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.665759243Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:15.709313256Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.709912985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.741808945Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.742694311Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.742736203Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.74275268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.742770297Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966} {"msg":"Crucible fc8bc925-cfa9-43e0-8b7c-eab69bd73542 has session id: 699283eb-98bf-4fda-a616-d2c73c28f66f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.743346541Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Upstairs opts: Upstairs UUID: fc8bc925-cfa9-43e0-8b7c-eab69bd73542, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.743381064Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Crucible stats registered with UUID: fc8bc925-cfa9-43e0-8b7c-eab69bd73542","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.743399181Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:15.743414598Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.743875023Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.743897748Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966} {"msg":"fc8bc925-cfa9-43e0-8b7c-eab69bd73542 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.743954548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.743981703Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"0","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744011447Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"0","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744025895Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"1","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.74404894Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"1","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744073876Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"2","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744099771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"2","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744115548Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744130036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744150692Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63760)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744388778Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53979)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.74443146Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46990)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.744459384Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.745043316Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"1","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.74507846Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"2","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.745232252Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"0","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.745743297Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"1","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.74578268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"2","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.745885141Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"0","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746337097Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746447017Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[0]R flush_numbers[0..12]: [319, 325, 328, 317, 307, 326, 328, 314, 328, 291, 307, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746565915Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[0]R generation[0..12]: [188, 190, 193, 187, 182, 191, 193, 186, 193, 174, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746586991Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746607517Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[1]R flush_numbers[0..12]: [319, 325, 328, 317, 307, 326, 328, 314, 328, 291, 307, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746622304Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[1]R generation[0..12]: [188, 190, 193, 187, 182, 191, 193, 186, 193, 174, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746635922Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746649739Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[2]R flush_numbers[0..12]: [319, 325, 328, 317, 307, 326, 328, 314, 328, 291, 307, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746663137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[2]R generation[0..12]: [188, 190, 193, 187, 182, 191, 193, 186, 193, 174, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746676385Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746689992Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:27:15.74670258Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746721776Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746734274Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746746861Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"":"downstairs","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746759879Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746771597Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"0","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746784914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"1","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746800322Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"client":"2","":"downstairs","session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746814099Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746826397Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"fc8bc925-cfa9-43e0-8b7c-eab69bd73542 is now active with session: 699283eb-98bf-4fda-a616-d2c73c28f66f","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746838815Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.746850632Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966,"session_id":"699283eb-98bf-4fda-a616-d2c73c28f66f"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:15.74686245Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 116 len 16384 data: 19 15 14 17 02/30 Read block 68 len 4096 03/30 Read block 208 len 24576 04/30 Write block 163 len 4096 data: 14 05/30 Write block 558 len 40960 data: 14 15 16 18 16 15 16 18 17 17 06/30 Read block 246 len 16384 07/30 Read block 522 len 12288 08/30 Flush 09/30 Write block 331 len 32768 data: 10 15 15 16 15 14 11 14 10/30 Read block 104 len 8192 11/30 Write block 325 len 28672 data: 13 14 15 11 9 8 11 12/30 Write block 338 len 24576 data: 15 18 15 12 14 14 13/30 Write block 288 len 40960 data: 15 17 19 19 13 14 11 13 9 13 14/30 Read block 274 len 32768 15/30 Read block 537 len 12288 16/30 Write block 514 len 16384 data: 13 13 14 13 17/30 Write block 223 len 4096 data: 10 18/30 Read block 241 len 8192 19/30 Read block 40 len 20480 20/30 Read block 296 len 40960 21/30 Read block 523 len 4096 22/30 Flush 23/30 Write block 62 len 4096 data: 13 24/30 Write block 591 len 8192 data: 7 7 25/30 Read block 203 len 36864 26/30 Write block 365 len 32768 data: 13 15 18 17 18 16 16 17 27/30 Write block 553 len 24576 data: 14 13 15 15 16 15 28/30 Read block 557 len 32768 29/30 Write block 104 len 28672 data: 10 11 14 17 21 20 20 30/30 Read block 9 len 20480 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:15.767410189Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 193 193 193 328 328 328 F F F 3 060-079 187 187 187 317 317 317 T T T 4 080-099 182 182 182 307 307 307 F F F 5 100-119 194 194 194 329 329 329 T T T 6 120-139 193 193 193 328 328 328 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 194 194 194 329 329 329 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 194 194 194 330 330 330 F F F 12 240-259 193 193 193 328 328 328 F F F 13 260-279 193 193 193 328 328 328 F F F 14 280-299 194 194 194 330 330 330 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 194 194 194 330 330 330 F F F 17 340-359 194 194 194 330 330 330 F F F 18 360-379 186 186 186 316 316 316 T T T 19 380-399 190 190 190 325 325 325 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 193 193 193 328 328 328 F F F 24 480-499 192 192 192 327 327 327 F F F 25 500-519 194 194 194 330 330 330 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 194 194 194 329 329 329 T T T 28 560-579 194 194 194 329 329 329 F F F 29 580-599 190 190 190 324 324 324 T T T Max gen: 194, Max flush: 330 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 193 193 193 328 328 328 F F F 3 060-079 195 195 195 331 331 331 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 195 195 195 331 331 331 F F F 6 120-139 193 193 193 328 328 328 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 194 194 194 329 329 329 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 182 182 182 307 307 307 F F F 11 220-239 194 194 194 330 330 330 F F F 12 240-259 193 193 193 328 328 328 F F F 13 260-279 193 193 193 328 328 328 F F F 14 280-299 194 194 194 330 330 330 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 194 194 194 330 330 330 F F F 17 340-359 194 194 194 330 330 330 F F F 18 360-379 195 195 195 331 331 331 F F F 19 380-399 190 190 190 325 325 325 F F F 20 400-419 184 184 184 312 312 312 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 192 192 192 327 327 327 F F F 23 460-479 193 193 193 328 328 328 F F F 24 480-499 192 192 192 327 327 327 F F F 25 500-519 194 194 194 330 330 330 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 195 195 195 331 331 331 F F F 28 560-579 194 194 194 329 329 329 F F F 29 580-599 195 195 195 331 331 331 F F F Max gen: 195, Max flush: 331 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.052895491Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:20.098246732Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.098827255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.131482034Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.132430799Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.13247593Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.132492997Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.132510334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974} {"msg":"Crucible cd0dd808-ca89-40ca-8394-31235b94c4f0 has session id: 6c804848-741e-4a14-9824-fdf46f318d8e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133239369Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Upstairs opts: Upstairs UUID: cd0dd808-ca89-40ca-8394-31235b94c4f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133272393Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Crucible stats registered with UUID: cd0dd808-ca89-40ca-8394-31235b94c4f0","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.13328821Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:20.133301198Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133792187Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133818502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974} {"msg":"cd0dd808-ca89-40ca-8394-31235b94c4f0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133893468Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133912114Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"0","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133945788Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"0","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133960266Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"1","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133974563Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"1","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.133996489Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"2","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.134023234Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"2","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.134049749Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.134073955Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.13410111Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47816)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.134240534Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36902)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.134284786Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34727)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.13431502Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.135099595Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"1","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.13512882Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"2","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.1352901Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"0","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.135815073Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"1","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.135844827Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"2","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.135966914Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"0","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136413332Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.13647522Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[0]R flush_numbers[0..12]: [319, 325, 328, 331, 307, 331, 328, 314, 329, 291, 307, 330]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136601097Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[0]R generation[0..12]: [188, 190, 193, 195, 182, 195, 193, 186, 194, 174, 182, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136622033Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136636641Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[1]R flush_numbers[0..12]: [319, 325, 328, 331, 307, 331, 328, 314, 329, 291, 307, 330]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136650478Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[1]R generation[0..12]: [188, 190, 193, 195, 182, 195, 193, 186, 194, 174, 182, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136663816Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136677323Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[2]R flush_numbers[0..12]: [319, 325, 328, 331, 307, 331, 328, 314, 329, 291, 307, 330]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136690631Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[2]R generation[0..12]: [188, 190, 193, 195, 182, 195, 193, 186, 194, 174, 182, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136703798Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136717416Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136735122Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":Wait for a query_work_queue command to finish before sending IO 1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136755339Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136768086Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136780464Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"":"downstairs","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136793812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136805709Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"0","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136818747Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"1","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136831694Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"client":"2","":"downstairs","session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136844982Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.13685699Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"cd0dd808-ca89-40ca-8394-31235b94c4f0 is now active with session: 6c804848-741e-4a14-9824-fdf46f318d8e","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136869818Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136881255Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974,"session_id":"6c804848-741e-4a14-9824-fdf46f318d8e"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:20.136893313Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 4096 data: 16 02/30 Write block 383 len 20480 data: 15 15 13 14 14 03/30 Write block 117 len 32768 data: 16 15 18 22 17 13 12 13 04/30 Read block 161 len 24576 05/30 Write block 209 len 8192 data: 13 11 06/30 Read block 587 len 4096 07/30 Read block 102 len 4096 08/30 Read block 112 len 20480 09/30 Read block 271 len 12288 10/30 Write block 353 len 40960 data: 18 19 20 23 24 24 25 20 19 16 11/30 Flush 12/30 Write block 456 len 32768 data: 18 20 18 16 13 13 14 14 13/30 Write block 288 len 32768 data: 16 18 20 20 14 15 12 14 14/30 Read block 112 len 28672 15/30 Read block 148 len 12288 16/30 Read block 234 len 8192 17/30 Write block 397 len 28672 data: 10 13 17 19 17 14 14 18/30 Read block 472 len 28672 19/30 Read block 248 len 4096 20/30 Read block 362 len 32768 21/30 Flush 22/30 Write block 54 len 28672 data: 14 12 11 11 8 7 9 23/30 Read block 341 len 20480 24/30 Read block 559 len 12288 25/30 Read block 333 len 8192 26/30 Read block 238 len 28672 27/30 Read block 493 len 12288 28/30 Read block 21 len 40960 29/30 Write block 193 len 24576 data: 17 19 17 17 16 14 30/30 Write block 598 len 8192 data: 4 3 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:20.157587985Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 193 193 193 328 328 328 T T T 3 060-079 195 195 195 331 331 331 T T T 4 080-099 182 182 182 307 307 307 F F F 5 100-119 196 196 196 332 332 332 F F F 6 120-139 196 196 196 332 332 332 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 194 194 194 329 329 329 F F F 9 180-199 196 196 196 332 332 332 T T T 10 200-219 196 196 196 332 332 332 F F F 11 220-239 194 194 194 330 330 330 F F F 12 240-259 193 193 193 328 328 328 F F F 13 260-279 193 193 193 328 328 328 F F F 14 280-299 196 196 196 333 333 333 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 194 194 194 330 330 330 F F F 17 340-359 196 196 196 332 332 332 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 196 196 196 333 333 333 F F F 20 400-419 196 196 196 333 333 333 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 196 196 196 333 333 333 F F F 23 460-479 196 196 196 333 333 333 F F F 24 480-499 192 192 192 327 327 327 F F F 25 500-519 194 194 194 330 330 330 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 195 195 195 331 331 331 F F F 28 560-579 194 194 194 329 329 329 F F F 29 580-599 195 195 195 331 331 331 T T T Max gen: 196, Max flush: 333 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 197 197 197 334 334 334 F F F 3 060-079 197 197 197 334 334 334 F F F 4 080-099 182 182 182 307 307 307 F F F 5 100-119 196 196 196 332 332 332 F F F 6 120-139 196 196 196 332 332 332 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 194 194 194 329 329 329 F F F 9 180-199 197 197 197 334 334 334 F F F 10 200-219 196 196 196 332 332 332 F F F 11 220-239 194 194 194 330 330 330 F F F 12 240-259 193 193 193 328 328 328 F F F 13 260-279 193 193 193 328 328 328 F F F 14 280-299 196 196 196 333 333 333 F F F 15 300-319 189 189 189 321 321 321 F F F 16 320-339 194 194 194 330 330 330 F F F 17 340-359 196 196 196 332 332 332 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 196 196 196 333 333 333 F F F 20 400-419 196 196 196 333 333 333 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 196 196 196 333 333 333 F F F 23 460-479 196 196 196 333 333 333 F F F 24 480-499 192 192 192 327 327 327 F F F 25 500-519 194 194 194 330 330 330 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 195 195 195 331 331 331 F F F 28 560-579 194 194 194 329 329 329 F F F 29 580-599 197 197 197 334 334 334 F F F Max gen: 197, Max flush: 334 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.437606379Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.482843041Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.483726648Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6c903058314cb23845d8ec9d1b70e497f207c60e\",\n git_commit_timestamp: \"2025-03-07T17:56:28.000000000Z\",\n git_branch: \"renovate/dropshot-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.48376951Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.483784317Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.483799494Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982} {"msg":"Crucible daea9b69-0959-4341-9747-31f71697a97d has session id: 00b19d6d-4f8e-4208-a285-f03dc3c3b2d2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.484378137Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Upstairs opts: Upstairs UUID: daea9b69-0959-4341-9747-31f71697a97d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.4844172Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Crucible stats registered with UUID: daea9b69-0959-4341-9747-31f71697a97d","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.484444175Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:24.484463561Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.484855109Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.484885743Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982} {"msg":"daea9b69-0959-4341-9747-31f71697a97d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.484937793Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.484964888Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"0","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.484994573Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"0","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.48501119Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"1","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485037735Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"1","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.48506335Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"2","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485090635Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"2","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485115651Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485139956Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485163822Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62560)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485188847Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58912)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485221971Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59967)","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485247386Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.485894767Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"0","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.48593099Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"1","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.486528419Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"0","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.486642698Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"1","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.487062771Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.487270812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.488127514Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"2","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.488653036Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"2","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[0]R flush_numbers[0..12]: [319, 325, 334, 334, 307, 332, 332, 314, 329, 334, 332, 330]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489178319Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[0]R generation[0..12]: [188, 190, 197, 197, 182, 196, 196, 186, 194, 197, 196, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489202495Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489220581Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[1]R flush_numbers[0..12]: [319, 325, 334, 334, 307, 332, 332, 314, 329, 334, 332, 330]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489238268Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[1]R generation[0..12]: [188, 190, 197, 197, 182, 196, 196, 186, 194, 197, 196, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489259334Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489276561Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[2]R flush_numbers[0..12]: [319, 325, 334, 334, 307, 332, 332, 314, 329, 334, 332, 330]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489293538Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[2]R generation[0..12]: [188, 190, 197, 197, 182, 196, 196, 186, 194, 197, 196, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489310175Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-07T18:27:24.489326892Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489347578Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489363615Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489379492Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489395069Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"":"downstairs","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489410806Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489425214Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"0","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489441271Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"1","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489457048Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"client":"2","":"downstairs","session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489473015Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489487812Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"daea9b69-0959-4341-9747-31f71697a97d is now active with session: 00b19d6d-4f8e-4208-a285-f03dc3c3b2d2","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489506039Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489522045Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982,"session_id":"00b19d6d-4f8e-4208-a285-f03dc3c3b2d2"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-07T18:27:24.489537093Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 86 len 24576 data: 12 15 14 13 11 11 02/30 Flush 03/30 Write block 473 len 8192 data: 13 17 04/30 Read block 216 len 28672 05/30 Write block 166 len 36864 data: 12 10 11 13 14 14 15 16 19 06/30 Read block 503 len 28672 07/30 Read block 264 len 20480 08/30 Write block 403 len 4096 data: 15 09/30 Read block 264 len 20480 10/30 Write block 292 len 12288 data: 15 16 13 11/30 Write block 318 len 12288 data: 20 21 21 12/30 Read block 245 len 12288 13/30 Write block 257 len 4096 data: 17 14/30 Read block 37 len 16384 15/30 Read block 412 len 24576 16/30 Write block 228 len 32768 data: 14 14 15 13 14 17 12 14 17/30 Write block 499 len 40960 data: 17 18 12 12 14 13 15 14 16 14 18/30 Read block 310 len 24576 19/30 Flush 20/30 Write block 109 len 28672 data: 21 21 17 15 19 18 19 21/30 Flush 22/30 Write block 41 len 20480 data: 11 10 7 11 12 23/30 Flush 24/30 Write block 170 len 16384 data: 15 15 16 17 25/30 Write block 451 len 32768 data: 12 11 15 14 16 19 21 19 26/30 Write block 563 len 20480 data: 16 17 19 18 18 27/30 Read block 11 len 12288 28/30 Write block 573 len 4096 data: 15 29/30 Write block 290 len 36864 data: 21 21 16 17 14 15 10 14 9 30/30 Read block 275 len 16384 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 9 Sent Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 5 5 7 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-07T18:27:24.509886878Z","hostname":"w-01JNRWVR4SNXKEADEZ76B93422","pid":1982} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 198 198 198 338 338 338 F F F 3 060-079 197 197 197 334 334 334 F F F 4 080-099 198 198 198 335 335 335 F F F 5 100-119 198 198 198 337 337 337 F F F 6 120-139 196 196 196 332 332 332 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 198 198 198 336 336 336 T T T 9 180-199 197 197 197 334 334 334 F F F 10 200-219 196 196 196 332 332 332 F F F 11 220-239 198 198 198 336 336 336 F F F 12 240-259 198 198 198 336 336 336 F F F 13 260-279 193 193 193 328 328 328 F F F 14 280-299 198 198 198 336 336 336 T T T 15 300-319 198 198 198 336 336 336 F F F 16 320-339 198 198 198 336 336 336 F F F 17 340-359 196 196 196 332 332 332 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 196 196 196 333 333 333 F F F 20 400-419 198 198 198 336 336 336 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 196 196 196 333 333 333 T T T 23 460-479 198 198 198 336 336 336 F F F 24 480-499 198 198 198 336 336 336 F F F 25 500-519 198 198 198 336 336 336 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 195 195 195 331 331 331 F F F 28 560-579 194 194 194 329 329 329 T T T 29 580-599 197 197 197 334 334 334 F F F Max gen: 198, Max flush: 338 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 319 319 319 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 198 198 198 338 338 338 F F F 3 060-079 197 197 197 334 334 334 F F F 4 080-099 198 198 198 335 335 335 F F F 5 100-119 198 198 198 337 337 337 F F F 6 120-139 196 196 196 332 332 332 F F F 7 140-159 186 186 186 314 314 314 F F F 8 160-179 199 199 199 339 339 339 F F F 9 180-199 197 197 197 334 334 334 F F F 10 200-219 196 196 196 332 332 332 F F F 11 220-239 198 198 198 336 336 336 F F F 12 240-259 198 198 198 336 336 336 F F F 13 260-279 193 193 193 328 328 328 F F F 14 280-299 199 199 199 339 339 339 F F F 15 300-319 198 198 198 336 336 336 F F F 16 320-339 198 198 198 336 336 336 F F F 17 340-359 196 196 196 332 332 332 F F F 18 360-379 196 196 196 332 332 332 F F F 19 380-399 196 196 196 333 333 333 F F F 20 400-419 198 198 198 336 336 336 F F F 21 420-439 191 191 191 326 326 326 F F F 22 440-459 199 199 199 339 339 339 F F F 23 460-479 198 198 198 336 336 336 F F F 24 480-499 198 198 198 336 336 336 F F F 25 500-519 198 198 198 336 336 336 F F F 26 520-539 184 184 184 312 312 312 F F F 27 540-559 195 195 195 331 331 331 F F F 28 560-579 199 199 199 339 339 339 F F F 29 580-599 197 197 197 334 334 334 F F F Max gen: 199, Max flush: 339 15:06 Test duration Test completed real 15:06.604666437 user 51.262862501 sys 20.884502773 trap 0.066429778 tflt 0.107598668 dflt 0.157650681 kflt 0.006996894 lock 24:16:21.710901101 slp 1:14:30.139685812 lat 5.886657568 stop 0.068714073